Attachment #13629 for bug #7419

View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All




package C4::ZebraIndex;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 2 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along with
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
# Suite 330, Boston, MA  02111-1307 USA

# This module contains utility functions to index records in Zebra
# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
# As soon as the new indexation layer is in place, and all scripts using this
# module are updated, this module have to be removed.

use Modern::Perl;
use POSIX;
use File::Temp qw/tempdir/;
use MARC::Record;

use C4::Context;
use C4::Biblio;
use C4::AuthoritiesMarc;
use C4::Items;
use Koha::RecordProcessor;

sub IndexRecord {
    my ($recordtype, $recordids, $options) = @_;

    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
    my $record_format = ($options and $options->{record_format})
                    ? $options->{record_format}
                    : ($as_xml)
                        ? 'marcxml'
                        : 'iso2709';
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
                        ? $options->{zebraidx_log_opt}
                        : "";
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
    my $directory = ($options and $options->{directory})
                    ? $options->{directory}
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));

    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
    @recordids = map { { biblio_auth_number => $_ } } @recordids;

    my $num_exported = 0;
    unless ($skip_export) {
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
    }
    unless ($skip_index) {
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
            $record_format, $zebraidx_log_opt);
    }

    return $num_exported;
}

sub DeleteRecordIndex {
    my ($recordtype, $recordids, $options) = @_;

    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
    my $record_format = ($options and $options->{record_format})
                    ? $options->{record_format}
                    : ($as_xml)
                        ? 'marcxml'
                        : 'iso2709';
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
                        ? $options->{zebraidx_log_opt}
                        : "";
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
    my $directory = ($options and $options->{directory})
                    ? $options->{directory}
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));

    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
    @recordids = map { { biblio_auth_number => $_ } } @recordids;

    my $num_exported = 0;
    unless ($skip_export) {
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
            $directory, $as_xml, $verbose);
    }
    unless ($skip_index) {
        _do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
            $record_format, $zebraidx_log_opt);
    }

    return $num_exported;
}


sub _export_marc_records_from_list {
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;

    my $num_exported = 0;
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
    if (_include_xml_wrapper($as_xml, $record_type)) {
        # include XML declaration and root element
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
    }
    my $i     = 0;
    my %found = ();
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
    foreach my $record_number (
        map  { $_->{biblio_auth_number} }
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
    ) {
        if($nosanitize) {
            my $marcxml = $record_type eq 'biblio'
                        ? GetXmlBiblio($record_number)
                        : GetAuthorityXML($record_number);
            if ($record_type eq 'biblio'){
                my @items = GetItemsInfo($record_number);
                if (@items){
                    my $record = MARC::Record->new;
                    $record->encoding('UTF-8');
                    my @itemsrecord;
                    foreach my $item (@items){
                        my $record = Item2Marc($item, $record_number);
                        push @itemsrecord, $record->field($itemtag);
                    }
                    $record->insert_fields_ordered(@itemsrecord);
                    my $itemsxml = $record->as_xml_record();
                    $marcxml =
                        substr($marcxml, 0, length($marcxml)-10) .
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
                }
            }
            if ($marcxml) {
                print {$fh} $marcxml;
                $num_exported++;
            }
        } else {
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
            if ( defined $marc ) {
                eval {
                    my $rec;
                    if ($as_xml) {
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
                    } else {
                        $rec = $marc->as_usmarc();
                    }
                    print {$fh} $rec;
                    $num_exported++;
                };
                if ($@) {
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
                }
            }
        }
        $i++;
        if($verbose) {
            print ".";
            print "$i\n" unless($i % 100);
        }
    }
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
    close $fh;
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
    return $num_exported;
}

sub _generate_deleted_marc_records {
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;

    my $num_exported = 0;
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
    if (_include_xml_wrapper($as_xml, $record_type)) {
        # include XML declaration and root element
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
    }
    my $i = 0;
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
        my $marc = MARC::Record->new();
        if ( $record_type eq 'biblio' ) {
            _fix_biblio_ids( $marc, $record_number, $record_number );
        } else {
            _fix_authority_id( $marc, $record_number );
        }
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
            _fix_unimarc_100($marc);
        }

        my $rec;
        if ($as_xml) {
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
        } else {
            $rec = $marc->as_usmarc();
        }
        print {$fh} $rec;

        $num_exported++;
        $i++;
        if($verbose) {
            print ".";
            print "$i\n" unless($i % 100);
        }
    }
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
    close $fh;

    return $num_exported;
}

sub _get_corrected_marc_record {
    my ( $record_type, $record_number, $noxml ) = @_;

    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );

    if ( defined $marc ) {
        _fix_leader($marc);
        if ( $record_type eq 'authority' ) {
            _fix_authority_id( $marc, $record_number );
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
            $marc = $normalizer->process($marc);
        }
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
            _fix_unimarc_100($marc);
        }
    }

    return $marc;
}

sub _get_raw_marc_record {
    my ( $record_type, $record_number, $noxml ) = @_;

    my $dbh = C4::Context->dbh;
    my $marc;
    if ( $record_type eq 'biblio' ) {
        if ($noxml) {
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
                WHERE biblionumber = ?");
            $fetch_sth->execute($record_number);
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
                $marc = MARC::Record->new_from_usmarc($blob);
                unless ($marc) {
                    warn "error creating MARC::Record from $blob";
                }
            }
            # failure to find a bib is not a problem -
            # a delete could have been done before
            # trying to process a record update

            $fetch_sth->finish();
            return unless $marc;
        } else {
            eval { $marc = GetMarcBiblio($record_number, 1); };
            if ($@ || !$marc) {
                # here we do warn since catching an exception
                # means that the bib was found but failed
                # to be parsed
                warn "error retrieving biblio $record_number: $@";
                return;
            }
        }
    } else {
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
        if ($@) {
            warn "error retrieving authority $record_number: $@";
            return;
        }
    }
    return $marc;
}

sub _fix_leader {

    # FIXME - this routine is suspect
    # It blanks the Leader/00-05 and Leader/12-16 to
    # force them to be recalculated correct when
    # the $marc->as_usmarc() or $marc->as_xml() is called.
    # But why is this necessary?  It would be a serious bug
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
    # if they are emitting incorrect leader values.
    my $marc = shift;

    my $leader = $marc->leader;
    substr( $leader, 0,  5 ) = '     ';
    substr( $leader, 10, 7 ) = '22     ';
    $marc->leader( substr( $leader, 0, 24 ) );
}


sub _fix_biblio_ids {

    # FIXME - it is essential to ensure that the biblionumber is present,
    #         otherwise, Zebra will choke on the record.  However, this
    #         logic belongs in the relevant C4::Biblio APIs.
    my $marc         = shift;
    my $biblionumber = shift;
    my $dbh = C4::Context->dbh;
    my $biblioitemnumber;
    if (@_) {
        $biblioitemnumber = shift;
    } else {
        my $sth = $dbh->prepare(
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
        $sth->execute($biblionumber);
        ($biblioitemnumber) = $sth->fetchrow_array;
        $sth->finish;
        unless ($biblioitemnumber) {
            warn "failed to get biblioitemnumber for biblio $biblionumber";
            return 0;
        }
    }

    # FIXME - this is cheating on two levels
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
    #
    # On the other hand, this better for now than what rebuild_zebra.pl used to
    # do, which was duplicate the code for inserting the biblionumber
    # and biblioitemnumber
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );

    return 1;
}

sub _fix_authority_id {

    # FIXME - as with fix_biblio_ids, the authid must be present
    #         for Zebra's sake.  However, this really belongs
    #         in C4::AuthoritiesMarc.
    my ( $marc, $authid ) = @_;
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
        $marc->delete_field( $marc->field('001') );
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
    }
}

sub _fix_unimarc_100 {

    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
    my $marc = shift;

    my $string;
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
        $string = $marc->subfield( 100, "a" );
        my $f100 = $marc->field(100);
        $marc->delete_field($f100);
    } else {
        $string = POSIX::strftime( "%Y%m%d", localtime );
        $string =~ s/\-//g;
        $string = sprintf( "%-*s", 35, $string );
    }
    substr( $string, 22, 6, "frey50" );
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
        $marc->delete_field( $marc->field(100) );
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
    }
}

sub _do_indexing {
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;

    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};

    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;

}

sub _include_xml_wrapper {
    my $as_xml = shift;
    my $record_type = shift;

    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';

    return 0 unless $as_xml;
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
    return 0;

}

1;




use C4::Biblio;
use C4::AuthoritiesMarc;
use C4::Items;
use C4::ZebraIndex;

# 
# script that checks zebradir structure & create directories & mandatory files if needed

}
if ($keep_export) {
    print "NOTHING cleaned : the export $directory has been kept.\n";
    print "You can re-run this script with the -s and -d $directory parameters";
    if ($use_tempdir) {
        print " and -d $directory parameters";
    } else {
        print "parameter";
    }
    print "\n";
    print "if you just want to rebuild zebra after changing the record.abs\n";
    print "or another zebra config file\n";

sub index_records {
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;

    my $num_records_exported = 0;
    my $records_deleted;
    my $need_reset = check_zebra_dirs($server_dir);
    if ($need_reset) {
        print "$0: found broken zebra server directories: forcing a rebuild\n";
        $reset = 1;
    }

    if ( $verbose_logging ) {
        print "====================\n";
        print "REINDEXING zebra\n";
        print "====================\n";
    } else {
        if ( $verbose_logging ) {
            print "====================\n";
            print "exporting $record_type\n";
            print "====================\n";
        }
        mkdir "$directory" unless (-d $directory);
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
        if ($process_zebraqueue) {
            my $entries = select_zebraqueue_records($record_type, 'deleted');
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
            mark_zebraqueue_batch_done($entries);
            $entries = select_zebraqueue_records($record_type, 'updated');
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
            $num_records_exported = export_marc_records_from_list($record_type, 
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
            mark_zebraqueue_batch_done($entries);
        } else {
            my $sth = select_all_records($record_type);
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
            unless ($do_not_clear_zebraqueue) {
                mark_all_zebraqueue_done($record_type);
            }
        }
    }

    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
    if ($process_zebraqueue) {
        # Process 'deleted' records
        my $entries = select_zebraqueue_records( $record_type, 'deleted' );
        my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;

        C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
            as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
            skip_export => $skip_export, skip_index => $skip_index,
            keep_export => $keep_export, directory => $directory
        });

        mark_zebraqueue_batch_done($entries);

        # Process 'updated' records'
        $entries = select_zebraqueue_records( $record_type, 'updated' );
        my @entries_to_update = map {
            my $id = $_->{biblio_auth_id};
            # do not try to update deleted records
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
        } @$entries;

        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
            noshadow => $noshadow, record_format => $record_fmt,
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
            skip_export => $skip_export, skip_index => $skip_index,
            keep_export => $keep_export, directory => $directory
        });

        mark_zebraqueue_batch_done($entries);
    } else {
        my $sth = select_all_records($record_type);
        my $entries = $sth->fetchall_arrayref([]);
        my @entries_to_update = map { $_->[0] } @$entries;

        C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
            noshadow => $noshadow, record_format => $record_fmt,
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
            verbose => $verbose_logging, skip_export => $skip_export,
            skip_index => $skip_index, keep_export => $keep_export,
            directory => $directory
        });

        unless ($do_not_clear_zebraqueue) {
            mark_all_zebraqueue_done($record_type);
        }
    }
}

    return $sth;
}

sub include_xml_wrapper {
    my $as_xml = shift;
    my $record_type = shift;

    return 0 unless $as_xml;
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
    return 0;

}

sub export_marc_records_from_sth {
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;

    my $num_exported = 0;
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
    if (include_xml_wrapper($as_xml, $record_type)) {
        # include XML declaration and root element
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
    }
    my $i = 0;
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
    while (my ($record_number) = $sth->fetchrow_array) {
        print "." if ( $verbose_logging );
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
        if ( $nosanitize ) {
            my $marcxml = $record_type eq 'biblio'
                          ? GetXmlBiblio( $record_number )
                          : GetAuthorityXML( $record_number );
            if ($record_type eq 'biblio'){
                my @items = GetItemsInfo($record_number);
                if (@items){
                    my $record = MARC::Record->new;
                    $record->encoding('UTF-8');
                    my @itemsrecord;
                    foreach my $item (@items){
                        my $record = Item2Marc($item, $record_number);                        
                        push @itemsrecord, $record->field($itemtag);
                    }
                    $record->insert_fields_ordered(@itemsrecord);
                    my $itemsxml = $record->as_xml_record();
                    $marcxml =
                        substr($marcxml, 0, length($marcxml)-10) .
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
                }
            }
            if ( $marcxml ) {
                print {$fh} $marcxml if $marcxml;
                $num_exported++;
            }
            next;
        }
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
        if (defined $marc) {
            eval {
                my $rec;
                if ($as_xml) {
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
                } else {
                    $rec = $marc->as_usmarc();
                }
                print {$fh} $rec;
                $num_exported++;
            };
            if ($@) {
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
            }
        }
    }
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
    close $fh;
    return $num_exported;
}

sub export_marc_records_from_list {
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;

    my $num_exported = 0;
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
    if (include_xml_wrapper($as_xml, $record_type)) {
        # include XML declaration and root element
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
    }
    my $i = 0;

    # Skip any deleted records. We check for this anyway, but this reduces error spam
    my %found = %$records_deleted;
    foreach my $record_number ( map { $_->{biblio_auth_number} }
                                grep { !$found{ $_->{biblio_auth_number} }++ }
                                @$entries ) {
        print "." if ( $verbose_logging );
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
        if (defined $marc) {
            eval {
                my $rec;
                if ($as_xml) {
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
                } else {
                    $rec = $marc->as_usmarc();
                }
                print {$fh} $rec;
                $num_exported++;
            };
            if ($@) {
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
            }
            $num_exported++;
        }
    }
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
    close $fh;
    return $num_exported;
}

sub generate_deleted_marc_records {
    my ($record_type, $entries, $directory, $as_xml) = @_;

    my $records_deleted = {};
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
    if (include_xml_wrapper($as_xml, $record_type)) {
        # include XML declaration and root element
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
    }
    my $i = 0;
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
        print "." if ( $verbose_logging );

        my $marc = MARC::Record->new();
        if ($record_type eq 'biblio') {
            fix_biblio_ids($marc, $record_number, $record_number);
        } else {
            fix_authority_id($marc, $record_number);
        }
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
            fix_unimarc_100($marc);
        }

        my $rec;
        if ($as_xml) {
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
        } else {
            $rec = $marc->as_usmarc();
        }
        print {$fh} $rec;

        $records_deleted->{$record_number} = 1;
    }
    print "\nRecords exported: $i\n" if ( $verbose_logging );
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
    close $fh;
    return $records_deleted;
    

}

sub get_corrected_marc_record {
    my ($record_type, $record_number, $noxml) = @_;

    my $marc = get_raw_marc_record($record_type, $record_number, $noxml); 

    if (defined $marc) {
        fix_leader($marc);
        if ($record_type eq 'authority') {
            fix_authority_id($marc, $record_number);
        } elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
            my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
            $marc = $normalizer->process($marc);
        }
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
            fix_unimarc_100($marc);
        }
    }

    return $marc;
}

sub get_raw_marc_record {
    my ($record_type, $record_number, $noxml) = @_;
  
    my $marc; 
    if ($record_type eq 'biblio') {
        if ($noxml) {
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
            $fetch_sth->execute($record_number);
            if (my ($blob) = $fetch_sth->fetchrow_array) {
                $marc = MARC::Record->new_from_usmarc($blob);
                unless ($marc) {
                    warn "error creating MARC::Record from $blob";
                }
            }
            # failure to find a bib is not a problem -
            # a delete could have been done before
            # trying to process a record update

            $fetch_sth->finish();
            return unless $marc;
        } else {
            eval { $marc = GetMarcBiblio($record_number, 1); };
            if ($@ || !$marc) {
                # here we do warn since catching an exception
                # means that the bib was found but failed
                # to be parsed
                warn "error retrieving biblio $record_number";
                return;
            }
        }
    } else {
        eval { $marc = GetAuthority($record_number); };
        if ($@) {
            warn "error retrieving authority $record_number";
            return;
        }
    }
    return $marc;
}

sub fix_leader {
    # FIXME - this routine is suspect
    # It blanks the Leader/00-05 and Leader/12-16 to
    # force them to be recalculated correct when
    # the $marc->as_usmarc() or $marc->as_xml() is called.
    # But why is this necessary?  It would be a serious bug
    # in MARC::Record (definitely) and MARC::File::XML (arguably) 
    # if they are emitting incorrect leader values.
    my $marc = shift;

    my $leader = $marc->leader;
    substr($leader,  0, 5) = '     ';
    substr($leader, 10, 7) = '22     ';
    $marc->leader(substr($leader, 0, 24));
}

sub fix_biblio_ids {
    # FIXME - it is essential to ensure that the biblionumber is present,
    #         otherwise, Zebra will choke on the record.  However, this
    #         logic belongs in the relevant C4::Biblio APIs.
    my $marc = shift;
    my $biblionumber = shift;
    my $biblioitemnumber;
    if (@_) {
        $biblioitemnumber = shift;
    } else {    
        my $sth = $dbh->prepare(
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
        $sth->execute($biblionumber);
        ($biblioitemnumber) = $sth->fetchrow_array;
        $sth->finish;
        unless ($biblioitemnumber) {
            warn "failed to get biblioitemnumber for biblio $biblionumber";
            return 0;
        }
    }

    # FIXME - this is cheating on two levels
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
    #
    # On the other hand, this better for now than what rebuild_zebra.pl used to
    # do, which was duplicate the code for inserting the biblionumber 
    # and biblioitemnumber
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);

    return 1;
}

sub fix_authority_id {
    # FIXME - as with fix_biblio_ids, the authid must be present
    #         for Zebra's sake.  However, this really belongs
    #         in C4::AuthoritiesMarc.
    my ($marc, $authid) = @_;
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
        $marc->delete_field($marc->field('001'));
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
    }
}

sub fix_unimarc_100 {
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
    my $marc = shift;

    my $string;
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
        $string = $marc->subfield( 100, "a" );
        my $f100 = $marc->field(100);
        $marc->delete_field($f100);
    }
    else {
        $string = POSIX::strftime( "%Y%m%d", localtime );
        $string =~ s/\-//g;
        $string = sprintf( "%-*s", 35, $string );
    }
    substr( $string, 22, 6, "frey50" );
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
        $marc->delete_field($marc->field(100));
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
    }
}

sub do_indexing {
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;

    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};

    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;

}

sub print_usage {
    print <<_USAGE_;
- 

Return to bug 7419

Line 0 Link Here

(-)a/C4/ZebraIndex.pm (+407 lines)
		1	package C4::ZebraIndex;
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it under the
		6	# terms of the GNU General Public License as published by the Free Software
		7	# Foundation; either version 2 of the License, or (at your option) any later
		8	# version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		13	#
		14	# You should have received a copy of the GNU General Public License along with
		15	# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
		16	# Suite 330, Boston, MA 02111-1307 USA
		17
		18	# This module contains utility functions to index records in Zebra
		19	# It is a temporary place to store rebuild_zebra.pl subs in order for them to be
		20	# used by other scripts (misc/migration_tools/dedup_authorities.pl for example)
		21	# As soon as the new indexation layer is in place, and all scripts using this
		22	# module are updated, this module have to be removed.
		23
		24	use Modern::Perl;
		25	use POSIX;
		26	use File::Temp qw/tempdir/;
		27	use MARC::Record;
		28
		29	use C4::Context;
		30	use C4::Biblio;
		31	use C4::AuthoritiesMarc;
		32	use C4::Items;
		33	use Koha::RecordProcessor;
		34
		35	sub IndexRecord {
		36	my ($recordtype, $recordids, $options) = @_;
		37
		38	my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
		39	my $noxml = ($options and $options->{noxml}) ? 1 : 0;
		40	my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
		41	my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
		42	my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
		43	my $record_format = ($options and $options->{record_format})
		44	? $options->{record_format}
		45	: ($as_xml)
		46	? 'marcxml'
		47	: 'iso2709';
		48	my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
		49	? $options->{zebraidx_log_opt}
		50	: "";
		51	my $verbose = ($options and $options->{verbose}) ? 1 : 0;
		52	my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
		53	my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
		54	my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
		55	my $directory = ($options and $options->{directory})
		56	? $options->{directory}
		57	: tempdir(CLEANUP => ($keep_export ? 0 : 1));
		58
		59	my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
		60	@recordids = map { { biblio_auth_number => $_ } } @recordids;
		61
		62	my $num_exported = 0;
		63	unless ($skip_export) {
		64	$num_exported = _export_marc_records_from_list($recordtype, \@recordids,
65	$directory, $as_xml, $noxml, $nosanitize, $verbose);
66	}
67	unless ($skip_index) {
68	_do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
69	$record_format, $zebraidx_log_opt);
70	}
71
72	return $num_exported;
73	}
74
75	sub DeleteRecordIndex {
76	my ($recordtype, $recordids, $options) = @_;
77
78	my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
79	my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
80	my $record_format = ($options and $options->{record_format})
81	? $options->{record_format}
82	: ($as_xml)
83	? 'marcxml'
84	: 'iso2709';
85	my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
86	? $options->{zebraidx_log_opt}
87	: "";
88	my $verbose = ($options and $options->{verbose}) ? 1 : 0;
89	my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
90	my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
91	my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
92	my $directory = ($options and $options->{directory})
93	? $options->{directory}
94	: tempdir(CLEANUP => ($keep_export ? 0 : 1));
95
96	my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
97	@recordids = map { { biblio_auth_number => $_ } } @recordids;
98
99	my $num_exported = 0;
100	unless ($skip_export) {
101	$num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
102	$directory, $as_xml, $verbose);
103	}
104	unless ($skip_index) {
105	_do_indexing($recordtype, "adelete", $directory, 0, $noshadow,
106	$record_format, $zebraidx_log_opt);
107	}
108
109	return $num_exported;
110	}
111
112
113	sub _export_marc_records_from_list {
114	my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
115
116	my $num_exported = 0;
117	open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
118	if (_include_xml_wrapper($as_xml, $record_type)) {
119	# include XML declaration and root element
120	print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
121	}
122	my $i = 0;
123	my %found = ();
124	my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
125	foreach my $record_number (
126	map { $_->{biblio_auth_number} }
127	grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
128	) {
129	if($nosanitize) {
130	my $marcxml = $record_type eq 'biblio'
131	? GetXmlBiblio($record_number)
132	: GetAuthorityXML($record_number);
133	if ($record_type eq 'biblio'){
134	my @items = GetItemsInfo($record_number);
135	if (@items){
136	my $record = MARC::Record->new;
137	$record->encoding('UTF-8');
138	my @itemsrecord;
139	foreach my $item (@items){
140	my $record = Item2Marc($item, $record_number);
141	push @itemsrecord, $record->field($itemtag);
142	}
143	$record->insert_fields_ordered(@itemsrecord);
144	my $itemsxml = $record->as_xml_record();
145	$marcxml =
146	substr($marcxml, 0, length($marcxml)-10) .
147	substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
148	}
149	}
150	if ($marcxml) {
151	print {$fh} $marcxml;
152	$num_exported++;
153	}
154	} else {
155	my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
156	if ( defined $marc ) {
157	eval {
158	my $rec;
159	if ($as_xml) {
160	$rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
161	$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
162	} else {
163	$rec = $marc->as_usmarc();
164	}
165	print {$fh} $rec;
166	$num_exported++;
167	};
168	if ($@) {
169	warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
170	}
171	}
172	}
173	$i++;
174	if($verbose) {
175	print ".";
176	print "$i\n" unless($i % 100);
177	}
178	}
179	print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
180	close $fh;
181	print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
182	return $num_exported;
183	}
184
185	sub _generate_deleted_marc_records {
186	my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
187
188	my $num_exported = 0;
189	open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
190	if (_include_xml_wrapper($as_xml, $record_type)) {
191	# include XML declaration and root element
192	print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
193	}
194	my $i = 0;
195	foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
196	my $marc = MARC::Record->new();
197	if ( $record_type eq 'biblio' ) {
198	_fix_biblio_ids( $marc, $record_number, $record_number );
199	} else {
200	_fix_authority_id( $marc, $record_number );
201	}
202	if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
203	_fix_unimarc_100($marc);
204	}
205
206	my $rec;
207	if ($as_xml) {
208	$rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
209	$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
210	} else {
211	$rec = $marc->as_usmarc();
212	}
213	print {$fh} $rec;
214
215	$num_exported++;
216	$i++;
217	if($verbose) {
218	print ".";
219	print "$i\n" unless($i % 100);
220	}
221	}
222	print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
223	close $fh;
224
225	return $num_exported;
226	}
227
228	sub _get_corrected_marc_record {
229	my ( $record_type, $record_number, $noxml ) = @_;
230
231	my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
232
233	if ( defined $marc ) {
234	_fix_leader($marc);
235	if ( $record_type eq 'authority' ) {
236	_fix_authority_id( $marc, $record_number );
237	} elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
238	my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
239	$marc = $normalizer->process($marc);
240	}
241	if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
242	_fix_unimarc_100($marc);
243	}
244	}
245
246	return $marc;
247	}
248
249	sub _get_raw_marc_record {
250	my ( $record_type, $record_number, $noxml ) = @_;
251
252	my $dbh = C4::Context->dbh;
253	my $marc;
254	if ( $record_type eq 'biblio' ) {
255	if ($noxml) {
256	my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
257	WHERE biblionumber = ?");
258	$fetch_sth->execute($record_number);
259	if ( my ($blob) = $fetch_sth->fetchrow_array ) {
260	$marc = MARC::Record->new_from_usmarc($blob);
261	unless ($marc) {
262	warn "error creating MARC::Record from $blob";
263	}
264	}
265	# failure to find a bib is not a problem -
266	# a delete could have been done before
267	# trying to process a record update
268
269	$fetch_sth->finish();
270	return unless $marc;
271	} else {
272	eval { $marc = GetMarcBiblio($record_number, 1); };
273	if ($@ \|\| !$marc) {
274	# here we do warn since catching an exception
275	# means that the bib was found but failed
276	# to be parsed
277	warn "error retrieving biblio $record_number: $@";
278	return;
279	}
280	}
281	} else {
282	eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
283	if ($@) {
284	warn "error retrieving authority $record_number: $@";
285	return;
286	}
287	}
288	return $marc;
289	}
290
291	sub _fix_leader {
292
293	# FIXME - this routine is suspect
294	# It blanks the Leader/00-05 and Leader/12-16 to
295	# force them to be recalculated correct when
296	# the $marc->as_usmarc() or $marc->as_xml() is called.
297	# But why is this necessary? It would be a serious bug
298	# in MARC::Record (definitely) and MARC::File::XML (arguably)
299	# if they are emitting incorrect leader values.
300	my $marc = shift;
301
302	my $leader = $marc->leader;
303	substr( $leader, 0, 5 ) = ' ';
304	substr( $leader, 10, 7 ) = '22 ';
305	$marc->leader( substr( $leader, 0, 24 ) );
306	}
307
308
309	sub _fix_biblio_ids {
310
311	# FIXME - it is essential to ensure that the biblionumber is present,
312	# otherwise, Zebra will choke on the record. However, this
313	# logic belongs in the relevant C4::Biblio APIs.
314	my $marc = shift;
315	my $biblionumber = shift;
316	my $dbh = C4::Context->dbh;
317	my $biblioitemnumber;
318	if (@_) {
319	$biblioitemnumber = shift;
320	} else {
321	my $sth = $dbh->prepare(
322	"SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
323	$sth->execute($biblionumber);
324	($biblioitemnumber) = $sth->fetchrow_array;
325	$sth->finish;
326	unless ($biblioitemnumber) {
327	warn "failed to get biblioitemnumber for biblio $biblionumber";
328	return 0;
329	}
330	}
331
332	# FIXME - this is cheating on two levels
333	# 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
334	# 2. Making sure that the biblionumber and biblioitemnumber are correct and
335	# present in the MARC::Record object ought to be part of GetMarcBiblio.
336	#
337	# On the other hand, this better for now than what rebuild_zebra.pl used to
338	# do, which was duplicate the code for inserting the biblionumber
339	# and biblioitemnumber
340	C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
341
342	return 1;
343	}
344
345	sub _fix_authority_id {
346
347	# FIXME - as with fix_biblio_ids, the authid must be present
348	# for Zebra's sake. However, this really belongs
349	# in C4::AuthoritiesMarc.
350	my ( $marc, $authid ) = @_;
351	unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
352	$marc->delete_field( $marc->field('001') );
353	$marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
354	}
355	}
356
357	sub _fix_unimarc_100 {
358
359	# FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
360	my $marc = shift;
361
362	my $string;
363	if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
364	$string = $marc->subfield( 100, "a" );
365	my $f100 = $marc->field(100);
366	$marc->delete_field($f100);
367	} else {
368	$string = POSIX::strftime( "%Y%m%d", localtime );
369	$string =~ s/\-//g;
370	$string = sprintf( "%-*s", 35, $string );
371	}
372	substr( $string, 22, 6, "frey50" );
373	unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
374	$marc->delete_field( $marc->field(100) );
375	$marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
376	}
377	}
378
379	sub _do_indexing {
380	my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
381
382	my $zebra_server = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
383	my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios' : 'authorities';
384	my $zebra_config = C4::Context->zebraconfig($zebra_server)->{'config'};
385	my $zebra_db_dir = C4::Context->zebraconfig($zebra_server)->{'directory'};
386
387	system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
388	system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
389	system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
390
391	}
392
393	sub _include_xml_wrapper {
394	my $as_xml = shift;
395	my $record_type = shift;
396
397	my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') \|\| 'grs1';
398	my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') \|\| 'dom';
399
400	return 0 unless $as_xml;
401	return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
402	return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
403	return 0;
404
405	}
406
407	1;

Lines 10-16 use File::Path; Link Here

(-)a/misc/migration_tools/rebuild_zebra.pl (-381 / +54 lines)
10	use C4::Biblio;	10	use C4::Biblio;
11	use C4::AuthoritiesMarc;	11	use C4::AuthoritiesMarc;
12	use C4::Items;	12	use C4::Items;
13	use Koha::RecordProcessor;	13	use C4::ZebraIndex;
14		14
15	#	15	#
16	# script that checks zebradir structure & create directories & mandatory files if needed	16	# script that checks zebradir structure & create directories & mandatory files if needed
Lines 156-167 if ( $verbose_logging ) { Link Here
156	}	156	}
157	if ($keep_export) {	157	if ($keep_export) {
158	print "NOTHING cleaned : the export $directory has been kept.\n";	158	print "NOTHING cleaned : the export $directory has been kept.\n";
159	print "You can re-run this script with the -s ";	159	print "You can re-run this script with the -s and -d $directory parameters";
160	if ($use_tempdir) {
161	print " and -d $directory parameters";
162	} else {
163	print "parameter";
164	}
165	print "\n";	160	print "\n";
166	print "if you just want to rebuild zebra after changing the record.abs\n";	161	print "if you just want to rebuild zebra after changing the record.abs\n";
167	print "or another zebra config file\n";	162	print "or another zebra config file\n";
Lines 196-263 sub check_zebra_dirs { Link Here
196	sub index_records {	191	sub index_records {
197	my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;	192	my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
198		193
199	my $num_records_exported = 0;
200	my $records_deleted;
201	my $need_reset = check_zebra_dirs($server_dir);	194	my $need_reset = check_zebra_dirs($server_dir);
202	if ($need_reset) {	195	if ($need_reset) {
203	print "$0: found broken zebra server directories: forcing a rebuild\n";	196	print "$0: found broken zebra server directories: forcing a rebuild\n";
204	$reset = 1;	197	$reset = 1;
205	}	198	}
206	if ($skip_export && $verbose_logging) {	199
		200	if ( $verbose_logging ) {
207	print "====================\n";	201	print "====================\n";
208	print "SKIPPING $record_type export\n";	202	print "REINDEXING zebra\n";
209	print "====================\n";	203	print "====================\n";
210	} else {
211	if ( $verbose_logging ) {
212	print "====================\n";
213	print "exporting $record_type\n";
214	print "====================\n";
215	}
216	mkdir "$directory" unless (-d $directory);
217	mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
218	if ($process_zebraqueue) {
219	my $entries = select_zebraqueue_records($record_type, 'deleted');
220	mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
221	$records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
222	mark_zebraqueue_batch_done($entries);
223	$entries = select_zebraqueue_records($record_type, 'updated');
224	mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
225	$num_records_exported = export_marc_records_from_list($record_type,
226	$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
227	mark_zebraqueue_batch_done($entries);
228	} else {
229	my $sth = select_all_records($record_type);
230	$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
231	unless ($do_not_clear_zebraqueue) {
232	mark_all_zebraqueue_done($record_type);
233	}
234	}
235	}	204	}
236		205
237	#	206	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
238	# and reindexing everything	207	if ($process_zebraqueue) {
239	#	208	# Process 'deleted' records
240	if ($skip_index) {	209	my $entries = select_zebraqueue_records( $record_type, 'deleted' );
241	if ($verbose_logging) {	210	my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
242	print "====================\n";	211
243	print "SKIPPING $record_type indexing\n";	212	C4::ZebraIndex::DeleteRecordIndex($record_type, \@entries_to_delete, {
244	print "====================\n";	213	as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
245	}	214	zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
		215	skip_export => $skip_export, skip_index => $skip_index,
		216	keep_export => $keep_export, directory => $directory
		217	});
		218
		219	mark_zebraqueue_batch_done($entries);
		220
		221	# Process 'updated' records'
		222	$entries = select_zebraqueue_records( $record_type, 'updated' );
		223	my @entries_to_update = map {
		224	my $id = $_->{biblio_auth_id};
		225	# do not try to update deleted records
		226	(0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
		227	} @$entries;
		228
		229	C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
		230	as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
		231	noshadow => $noshadow, record_format => $record_fmt,
		232	zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
		233	skip_export => $skip_export, skip_index => $skip_index,
		234	keep_export => $keep_export, directory => $directory
		235	});
		236
		237	mark_zebraqueue_batch_done($entries);
246	} else {	238	} else {
247	if ( $verbose_logging ) {	239	my $sth = select_all_records($record_type);
248	print "====================\n";	240	my $entries = $sth->fetchall_arrayref([]);
249	print "REINDEXING zebra\n";	241	my @entries_to_update = map { $_->[0] } @$entries;
250	print "====================\n";	242
251	}	243	C4::ZebraIndex::IndexRecord($record_type, \@entries_to_update, {
252	my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;	244	as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
253	if ($process_zebraqueue) {	245	noshadow => $noshadow, record_format => $record_fmt,
254	do_indexing($record_type, 'adelete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)	246	zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
255	if %$records_deleted;	247	verbose => $verbose_logging, skip_export => $skip_export,
256	do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)	248	skip_index => $skip_index, keep_export => $keep_export,
257	if $num_records_exported;	249	directory => $directory
258	} else {	250	});
259	do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)	251
260	if ($num_records_exported or $skip_export);	252	unless ($do_not_clear_zebraqueue) {
		253	mark_all_zebraqueue_done($record_type);
261	}	254	}
262	}	255	}
263	}	256	}
Lines 327-651 sub select_all_biblios { Link Here
327	return $sth;	320	return $sth;
328	}	321	}
329		322
330	sub include_xml_wrapper {
331	my $as_xml = shift;
332	my $record_type = shift;
333
334	return 0 unless $as_xml;
335	return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
336	return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
337	return 0;
338
339	}
340
341	sub export_marc_records_from_sth {
342	my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
343
344	my $num_exported = 0;
345	open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
346	if (include_xml_wrapper($as_xml, $record_type)) {
347	# include XML declaration and root element
348	print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
349	}
350	my $i = 0;
351	my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
352	while (my ($record_number) = $sth->fetchrow_array) {
353	print "." if ( $verbose_logging );
354	print "\r$i" unless ($i++ %100 or !$verbose_logging);
355	if ( $nosanitize ) {
356	my $marcxml = $record_type eq 'biblio'
357	? GetXmlBiblio( $record_number )
358	: GetAuthorityXML( $record_number );
359	if ($record_type eq 'biblio'){
360	my @items = GetItemsInfo($record_number);
361	if (@items){
362	my $record = MARC::Record->new;
363	$record->encoding('UTF-8');
364	my @itemsrecord;
365	foreach my $item (@items){
366	my $record = Item2Marc($item, $record_number);
367	push @itemsrecord, $record->field($itemtag);
368	}
369	$record->insert_fields_ordered(@itemsrecord);
370	my $itemsxml = $record->as_xml_record();
371	$marcxml =
372	substr($marcxml, 0, length($marcxml)-10) .
373	substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
374	}
375	}
376	if ( $marcxml ) {
377	print {$fh} $marcxml if $marcxml;
378	$num_exported++;
379	}
380	next;
381	}
382	my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
383	if (defined $marc) {
384	eval {
385	my $rec;
386	if ($as_xml) {
387	$rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
388	$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
389	} else {
390	$rec = $marc->as_usmarc();
391	}
392	print {$fh} $rec;
393	$num_exported++;
394	};
395	if ($@) {
396	warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
397	}
398	}
399	}
400	print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
401	print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
402	close $fh;
403	return $num_exported;
404	}
405
406	sub export_marc_records_from_list {
407	my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
408
409	my $num_exported = 0;
410	open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
411	if (include_xml_wrapper($as_xml, $record_type)) {
412	# include XML declaration and root element
413	print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
414	}
415	my $i = 0;
416
417	# Skip any deleted records. We check for this anyway, but this reduces error spam
418	my %found = %$records_deleted;
419	foreach my $record_number ( map { $_->{biblio_auth_number} }
420	grep { !$found{ $_->{biblio_auth_number} }++ }
421	@$entries ) {
422	print "." if ( $verbose_logging );
423	print "\r$i" unless ($i++ %100 or !$verbose_logging);
424	my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
425	if (defined $marc) {
426	eval {
427	my $rec;
428	if ($as_xml) {
429	$rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
430	$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
431	} else {
432	$rec = $marc->as_usmarc();
433	}
434	print {$fh} $rec;
435	$num_exported++;
436	};
437	if ($@) {
438	warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
439	}
440	$num_exported++;
441	}
442	}
443	print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
444	print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
445	close $fh;
446	return $num_exported;
447	}
448
449	sub generate_deleted_marc_records {
450	my ($record_type, $entries, $directory, $as_xml) = @_;
451
452	my $records_deleted = {};
453	open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
454	if (include_xml_wrapper($as_xml, $record_type)) {
455	# include XML declaration and root element
456	print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
457	}
458	my $i = 0;
459	foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
460	print "\r$i" unless ($i++ %100 or !$verbose_logging);
461	print "." if ( $verbose_logging );
462
463	my $marc = MARC::Record->new();
464	if ($record_type eq 'biblio') {
465	fix_biblio_ids($marc, $record_number, $record_number);
466	} else {
467	fix_authority_id($marc, $record_number);
468	}
469	if (C4::Context->preference("marcflavour") eq "UNIMARC") {
470	fix_unimarc_100($marc);
471	}
472
473	my $rec;
474	if ($as_xml) {
475	$rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
476	$rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
477	} else {
478	$rec = $marc->as_usmarc();
479	}
480	print {$fh} $rec;
481
482	$records_deleted->{$record_number} = 1;
483	}
484	print "\nRecords exported: $i\n" if ( $verbose_logging );
485	print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
486	close $fh;
487	return $records_deleted;
488
489
490	}
491
492	sub get_corrected_marc_record {
493	my ($record_type, $record_number, $noxml) = @_;
494
495	my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
496
497	if (defined $marc) {
498	fix_leader($marc);
499	if ($record_type eq 'authority') {
500	fix_authority_id($marc, $record_number);
501	} elsif ($record_type eq 'biblio' && C4::Context->preference('IncludeSeeFromInSearches')) {
502	my $normalizer = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
503	$marc = $normalizer->process($marc);
504	}
505	if (C4::Context->preference("marcflavour") eq "UNIMARC") {
506	fix_unimarc_100($marc);
507	}
508	}
509
510	return $marc;
511	}
512
513	sub get_raw_marc_record {
514	my ($record_type, $record_number, $noxml) = @_;
515
516	my $marc;
517	if ($record_type eq 'biblio') {
518	if ($noxml) {
519	my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
520	$fetch_sth->execute($record_number);
521	if (my ($blob) = $fetch_sth->fetchrow_array) {
522	$marc = MARC::Record->new_from_usmarc($blob);
523	unless ($marc) {
524	warn "error creating MARC::Record from $blob";
525	}
526	}
527	# failure to find a bib is not a problem -
528	# a delete could have been done before
529	# trying to process a record update
530
531	$fetch_sth->finish();
532	return unless $marc;
533	} else {
534	eval { $marc = GetMarcBiblio($record_number, 1); };
535	if ($@ \|\| !$marc) {
536	# here we do warn since catching an exception
537	# means that the bib was found but failed
538	# to be parsed
539	warn "error retrieving biblio $record_number";
540	return;
541	}
542	}
543	} else {
544	eval { $marc = GetAuthority($record_number); };
545	if ($@) {
546	warn "error retrieving authority $record_number";
547	return;
548	}
549	}
550	return $marc;
551	}
552
553	sub fix_leader {
554	# FIXME - this routine is suspect
555	# It blanks the Leader/00-05 and Leader/12-16 to
556	# force them to be recalculated correct when
557	# the $marc->as_usmarc() or $marc->as_xml() is called.
558	# But why is this necessary? It would be a serious bug
559	# in MARC::Record (definitely) and MARC::File::XML (arguably)
560	# if they are emitting incorrect leader values.
561	my $marc = shift;
562
563	my $leader = $marc->leader;
564	substr($leader, 0, 5) = ' ';
565	substr($leader, 10, 7) = '22 ';
566	$marc->leader(substr($leader, 0, 24));
567	}
568
569	sub fix_biblio_ids {
570	# FIXME - it is essential to ensure that the biblionumber is present,
571	# otherwise, Zebra will choke on the record. However, this
572	# logic belongs in the relevant C4::Biblio APIs.
573	my $marc = shift;
574	my $biblionumber = shift;
575	my $biblioitemnumber;
576	if (@_) {
577	$biblioitemnumber = shift;
578	} else {
579	my $sth = $dbh->prepare(
580	"SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
581	$sth->execute($biblionumber);
582	($biblioitemnumber) = $sth->fetchrow_array;
583	$sth->finish;
584	unless ($biblioitemnumber) {
585	warn "failed to get biblioitemnumber for biblio $biblionumber";
586	return 0;
587	}
588	}
589
590	# FIXME - this is cheating on two levels
591	# 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
592	# 2. Making sure that the biblionumber and biblioitemnumber are correct and
593	# present in the MARC::Record object ought to be part of GetMarcBiblio.
594	#
595	# On the other hand, this better for now than what rebuild_zebra.pl used to
596	# do, which was duplicate the code for inserting the biblionumber
597	# and biblioitemnumber
598	C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
599
600	return 1;
601	}
602
603	sub fix_authority_id {
604	# FIXME - as with fix_biblio_ids, the authid must be present
605	# for Zebra's sake. However, this really belongs
606	# in C4::AuthoritiesMarc.
607	my ($marc, $authid) = @_;
608	unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
609	$marc->delete_field($marc->field('001'));
610	$marc->insert_fields_ordered(MARC::Field->new('001',$authid));
611	}
612	}
613
614	sub fix_unimarc_100 {
615	# FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
616	my $marc = shift;
617
618	my $string;
619	if ( length($marc->subfield( 100, "a" )) == 36 ) {
620	$string = $marc->subfield( 100, "a" );
621	my $f100 = $marc->field(100);
622	$marc->delete_field($f100);
623	}
624	else {
625	$string = POSIX::strftime( "%Y%m%d", localtime );
626	$string =~ s/\-//g;
627	$string = sprintf( "%-*s", 35, $string );
628	}
629	substr( $string, 22, 6, "frey50" );
630	unless ( length($marc->subfield( 100, "a" )) == 36 ) {
631	$marc->delete_field($marc->field(100));
632	$marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
633	}
634	}
635
636	sub do_indexing {
637	my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
638
639	my $zebra_server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
640	my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
641	my $zebra_config = C4::Context->zebraconfig($zebra_server)->{'config'};
642	my $zebra_db_dir = C4::Context->zebraconfig($zebra_server)->{'directory'};
643
644	system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
645	system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
646	system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
647
648	}
649		323
650	sub print_usage {	324	sub print_usage {
651	print <<_USAGE_;	325	print <<_USAGE_;
652	-