View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All

(-)a/C4/Search.pm (+373 lines)
Lines 34-39 use C4::Charset; Link Here
34
use YAML;
34
use YAML;
35
use URI::Escape;
35
use URI::Escape;
36
use Business::ISBN;
36
use Business::ISBN;
37
use File::Temp qw/tempdir/;
37
38
38
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
39
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
39
40
Lines 2712-2717 sub GetDistinctValues { Link Here
2712
   }
2713
   }
2713
}
2714
}
2714
2715
2716
sub IndexRecord {
2717
    my ($recordtype, $recordids, $options) = @_;
2718
2719
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
2720
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
2721
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
2722
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
2723
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
2724
    my $record_format = ($options and $options->{record_format})
2725
                    ? $options->{record_format}
2726
                    : ($as_xml)
2727
                        ? 'marcxml'
2728
                        : 'iso2709';
2729
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
2730
                        ? $options->{zebraidx_log_opt}
2731
                        : "";
2732
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
2733
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
2734
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
2735
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
2736
    my $directory = ($options and $options->{directory})
2737
                    ? $options->{directory}
2738
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
2739
2740
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
2741
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
2742
2743
    my $num_exported = 0;
2744
    unless ($skip_export) {
2745
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
2746
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
2747
    }
2748
    unless ($skip_index) {
2749
        _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
2750
            $record_format, $zebraidx_log_opt);
2751
    }
2752
2753
    return $num_exported;
2754
}
2755
2756
sub DeleteRecordIndex {
2757
    my ($recordtype, $recordids, $options) = @_;
2758
2759
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
2760
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
2761
    my $record_format = ($options and $options->{record_format})
2762
                    ? $options->{record_format}
2763
                    : ($as_xml)
2764
                        ? 'marcxml'
2765
                        : 'iso2709';
2766
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
2767
                        ? $options->{zebraidx_log_opt}
2768
                        : "";
2769
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
2770
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
2771
    my $skip_index = ($options and $options->{skip_index}) ? 1 : 0;
2772
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
2773
    my $directory = ($options and $options->{directory})
2774
                    ? $options->{directory}
2775
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
2776
2777
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
2778
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
2779
2780
    my $num_exported = 0;
2781
    unless ($skip_export) {
2782
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
2783
            $directory, $as_xml, $verbose);
2784
    }
2785
    unless ($skip_index) {
2786
        _do_indexing($recordtype, "delete", $directory, 0, $noshadow,
2787
            $record_format, $zebraidx_log_opt);
2788
    }
2789
2790
    return $num_exported;
2791
}
2792
2793
2794
sub _export_marc_records_from_list {
2795
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
2796
2797
    my $num_exported = 0;
2798
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
2799
    if (_include_xml_wrapper($as_xml, $record_type)) {
2800
        # include XML declaration and root element
2801
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
2802
    }
2803
    my $i     = 0;
2804
    my %found = ();
2805
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
2806
    foreach my $record_number (
2807
        map  { $_->{biblio_auth_number} }
2808
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
2809
    ) {
2810
        if($nosanitize) {
2811
            my $marcxml = $record_type eq 'biblio'
2812
                        ? GetXmlBiblio($record_number)
2813
                        : GetAuthorityXML($record_number);
2814
            if ($record_type eq 'biblio'){
2815
                my @items = GetItemsInfo($record_number);
2816
                if (@items){
2817
                    my $record = MARC::Record->new;
2818
                    $record->encoding('UTF-8');
2819
                    my @itemsrecord;
2820
                    foreach my $item (@items){
2821
                        my $record = Item2Marc($item, $record_number);
2822
                        push @itemsrecord, $record->field($itemtag);
2823
                    }
2824
                    $record->insert_fields_ordered(@itemsrecord);
2825
                    my $itemsxml = $record->as_xml_record();
2826
                    $marcxml =
2827
                        substr($marcxml, 0, length($marcxml)-10) .
2828
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
2829
                }
2830
            }
2831
            if ($marcxml) {
2832
                print {$fh} $marcxml;
2833
                $num_exported++;
2834
            }
2835
        } else {
2836
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
2837
            if ( defined $marc ) {
2838
                eval {
2839
                    my $rec;
2840
                    if ($as_xml) {
2841
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
2842
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
2843
                    } else {
2844
                        $rec = $marc->as_usmarc();
2845
                    }
2846
                    print {$fh} $rec;
2847
                    $num_exported++;
2848
                };
2849
                if ($@) {
2850
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
2851
                }
2852
            }
2853
        }
2854
        $i++;
2855
        if($verbose) {
2856
            print ".";
2857
            print "$i\n" unless($i % 100);
2858
        }
2859
    }
2860
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
2861
    close $fh;
2862
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
2863
    return $num_exported;
2864
}
2865
2866
sub _generate_deleted_marc_records {
2867
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
2868
2869
    my $num_exported = 0;
2870
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
2871
    if (_include_xml_wrapper($as_xml, $record_type)) {
2872
        # include XML declaration and root element
2873
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
2874
    }
2875
    my $i = 0;
2876
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
2877
        my $marc = MARC::Record->new();
2878
        if ( $record_type eq 'biblio' ) {
2879
            _fix_biblio_ids( $marc, $record_number, $record_number );
2880
        } else {
2881
            _fix_authority_id( $marc, $record_number );
2882
        }
2883
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
2884
            _fix_unimarc_100($marc);
2885
        }
2886
2887
        my $rec;
2888
        if ($as_xml) {
2889
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
2890
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
2891
        } else {
2892
            $rec = $marc->as_usmarc();
2893
        }
2894
        print {$fh} $rec;
2895
2896
        $num_exported++;
2897
        $i++;
2898
        if($verbose) {
2899
            print ".";
2900
            print "$i\n" unless($i % 100);
2901
        }
2902
    }
2903
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
2904
    close $fh;
2905
2906
    return $num_exported;
2907
}
2908
2909
sub _get_corrected_marc_record {
2910
    my ( $record_type, $record_number, $noxml ) = @_;
2911
2912
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
2913
2914
    if ( defined $marc ) {
2915
        _fix_leader($marc);
2916
        if ( $record_type eq 'biblio' ) {
2917
            my $succeeded = _fix_biblio_ids( $marc, $record_number );
2918
            return unless $succeeded;
2919
        } else {
2920
            _fix_authority_id( $marc, $record_number );
2921
        }
2922
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
2923
            _fix_unimarc_100($marc);
2924
        }
2925
    }
2926
2927
    return $marc;
2928
}
2929
2930
sub _get_raw_marc_record {
2931
    my ( $record_type, $record_number, $noxml ) = @_;
2932
2933
    my $dbh = C4::Context->dbh;
2934
    my $marc;
2935
    if ( $record_type eq 'biblio' ) {
2936
        if ($noxml) {
2937
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
2938
                WHERE biblionumber = ?");
2939
            $fetch_sth->execute($record_number);
2940
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
2941
                $marc = MARC::Record->new_from_usmarc($blob);
2942
                unless ($marc) {
2943
                    warn "error creating MARC::Record from $blob";
2944
                }
2945
            }
2946
            # failure to find a bib is not a problem -
2947
            # a delete could have been done before
2948
            # trying to process a record update
2949
2950
            $fetch_sth->finish();
2951
            return unless $marc;
2952
        } else {
2953
            eval { $marc = GetMarcBiblio($record_number, 1); };
2954
            if ($@ || !$marc) {
2955
                # here we do warn since catching an exception
2956
                # means that the bib was found but failed
2957
                # to be parsed
2958
                warn "error retrieving biblio $record_number: $@";
2959
                return;
2960
            }
2961
        }
2962
    } else {
2963
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
2964
        if ($@) {
2965
            warn "error retrieving authority $record_number: $@";
2966
            return;
2967
        }
2968
    }
2969
    return $marc;
2970
}
2971
2972
sub _fix_leader {
2973
2974
    # FIXME - this routine is suspect
2975
    # It blanks the Leader/00-05 and Leader/12-16 to
2976
    # force them to be recalculated correct when
2977
    # the $marc->as_usmarc() or $marc->as_xml() is called.
2978
    # But why is this necessary?  It would be a serious bug
2979
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
2980
    # if they are emitting incorrect leader values.
2981
    my $marc = shift;
2982
2983
    my $leader = $marc->leader;
2984
    substr( $leader, 0,  5 ) = '     ';
2985
    substr( $leader, 10, 7 ) = '22     ';
2986
    $marc->leader( substr( $leader, 0, 24 ) );
2987
}
2988
2989
2990
sub _fix_biblio_ids {
2991
2992
    # FIXME - it is essential to ensure that the biblionumber is present,
2993
    #         otherwise, Zebra will choke on the record.  However, this
2994
    #         logic belongs in the relevant C4::Biblio APIs.
2995
    my $marc         = shift;
2996
    my $biblionumber = shift;
2997
    my $dbh = C4::Context->dbh;
2998
    my $biblioitemnumber;
2999
    if (@_) {
3000
        $biblioitemnumber = shift;
3001
    } else {
3002
        my $sth = $dbh->prepare(
3003
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
3004
        $sth->execute($biblionumber);
3005
        ($biblioitemnumber) = $sth->fetchrow_array;
3006
        $sth->finish;
3007
        unless ($biblioitemnumber) {
3008
            warn "failed to get biblioitemnumber for biblio $biblionumber";
3009
            return 0;
3010
        }
3011
    }
3012
3013
    # FIXME - this is cheating on two levels
3014
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
3015
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
3016
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
3017
    #
3018
    # On the other hand, this better for now than what rebuild_zebra.pl used to
3019
    # do, which was duplicate the code for inserting the biblionumber
3020
    # and biblioitemnumber
3021
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
3022
3023
    return 1;
3024
}
3025
3026
sub _fix_authority_id {
3027
3028
    # FIXME - as with fix_biblio_ids, the authid must be present
3029
    #         for Zebra's sake.  However, this really belongs
3030
    #         in C4::AuthoritiesMarc.
3031
    my ( $marc, $authid ) = @_;
3032
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
3033
        $marc->delete_field( $marc->field('001') );
3034
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
3035
    }
3036
}
3037
3038
sub _fix_unimarc_100 {
3039
3040
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
3041
    my $marc = shift;
3042
3043
    my $string;
3044
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
3045
        $string = $marc->subfield( 100, "a" );
3046
        my $f100 = $marc->field(100);
3047
        $marc->delete_field($f100);
3048
    } else {
3049
        $string = POSIX::strftime( "%Y%m%d", localtime );
3050
        $string =~ s/\-//g;
3051
        $string = sprintf( "%-*s", 35, $string );
3052
    }
3053
    substr( $string, 22, 6, "frey50" );
3054
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
3055
        $marc->delete_field( $marc->field(100) );
3056
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
3057
    }
3058
}
3059
3060
sub _do_indexing {
3061
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
3062
3063
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
3064
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
3065
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
3066
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
3067
3068
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
3069
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
3070
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
3071
3072
}
3073
3074
sub _include_xml_wrapper {
3075
    my $as_xml = shift;
3076
    my $record_type = shift;
3077
3078
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
3079
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';
3080
3081
    return 0 unless $as_xml;
3082
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
3083
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
3084
    return 0;
3085
3086
}
3087
2715
3088
2716
END { }    # module clean-up code here (global destructor)
3089
END { }    # module clean-up code here (global destructor)
2717
3090
(-)a/misc/migration_tools/rebuild_zebra.pl (-377 / +54 lines)
Lines 10-15 use File::Path; Link Here
10
use C4::Biblio;
10
use C4::Biblio;
11
use C4::AuthoritiesMarc;
11
use C4::AuthoritiesMarc;
12
use C4::Items;
12
use C4::Items;
13
use File::Temp qw(tempdir);
13
14
14
# 
15
# 
15
# script that checks zebradir structure & create directories & mandatory files if needed
16
# script that checks zebradir structure & create directories & mandatory files if needed
Lines 155-166 if ( $verbose_logging ) { Link Here
155
}
156
}
156
if ($keep_export) {
157
if ($keep_export) {
157
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "You can re-run this script with the -s ";
159
    print "You can re-run this script with the -s and -d $directory parameters";
159
    if ($use_tempdir) {
160
        print " and -d $directory parameters";
161
    } else {
162
        print "parameter";
163
    }
164
    print "\n";
160
    print "\n";
165
    print "if you just want to rebuild zebra after changing the record.abs\n";
161
    print "if you just want to rebuild zebra after changing the record.abs\n";
166
    print "or another zebra config file\n";
162
    print "or another zebra config file\n";
Lines 195-262 sub check_zebra_dirs { Link Here
195
sub index_records {
191
sub index_records {
196
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
192
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
197
193
198
    my $num_records_exported = 0;
199
    my $records_deleted;
200
    my $need_reset = check_zebra_dirs($server_dir);
194
    my $need_reset = check_zebra_dirs($server_dir);
201
    if ($need_reset) {
195
    if ($need_reset) {
202
    	print "$0: found broken zebra server directories: forcing a rebuild\n";
196
        print "$0: found broken zebra server directories: forcing a rebuild\n";
203
    	$reset = 1;
197
        $reset = 1;
204
    }
198
    }
205
    if ($skip_export && $verbose_logging) {
199
200
    if ( $verbose_logging ) {
206
        print "====================\n";
201
        print "====================\n";
207
        print "SKIPPING $record_type export\n";
202
        print "REINDEXING zebra\n";
208
        print "====================\n";
203
        print "====================\n";
209
    } else {
210
        if ( $verbose_logging ) {
211
            print "====================\n";
212
            print "exporting $record_type\n";
213
            print "====================\n";
214
        }
215
        mkdir "$directory" unless (-d $directory);
216
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
217
        if ($process_zebraqueue) {
218
            my $entries = select_zebraqueue_records($record_type, 'deleted');
219
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
220
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
221
            mark_zebraqueue_batch_done($entries);
222
            $entries = select_zebraqueue_records($record_type, 'updated');
223
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
224
            $num_records_exported = export_marc_records_from_list($record_type, 
225
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
226
            mark_zebraqueue_batch_done($entries);
227
        } else {
228
            my $sth = select_all_records($record_type);
229
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
230
            unless ($do_not_clear_zebraqueue) {
231
                mark_all_zebraqueue_done($record_type);
232
            }
233
        }
234
    }
204
    }
235
205
236
    #
206
    my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
237
    # and reindexing everything
207
    if ($process_zebraqueue) {
238
    #
208
        # Process 'deleted' records
239
    if ($skip_index) {
209
        my $entries = select_zebraqueue_records( $record_type, 'deleted' );
240
        if ($verbose_logging) {
210
        my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
241
            print "====================\n";
211
242
            print "SKIPPING $record_type indexing\n";
212
        C4::Search::DeleteRecordIndex($record_type, \@entries_to_delete, {
243
            print "====================\n";
213
            as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
244
        }
214
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
215
            skip_export => $skip_export, skip_index => $skip_index,
216
            keep_export => $keep_export, directory => $directory
217
        });
218
219
        mark_zebraqueue_batch_done($entries);
220
221
        # Process 'updated' records'
222
        $entries = select_zebraqueue_records( $record_type, 'updated' );
223
        my @entries_to_update = map {
224
            my $id = $_->{biblio_auth_id};
225
            # do not try to update deleted records
226
            (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
227
        } @$entries;
228
229
        C4::Search::IndexRecord($record_type, \@entries_to_update, {
230
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
231
            noshadow => $noshadow, record_format => $record_fmt,
232
            zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
233
            skip_export => $skip_export, skip_index => $skip_index,
234
            keep_export => $keep_export, directory => $directory
235
        });
236
237
        mark_zebraqueue_batch_done($entries);
245
    } else {
238
    } else {
246
        if ( $verbose_logging ) {
239
        my $sth = select_all_records($record_type);
247
            print "====================\n";
240
        my $entries = $sth->fetchall_arrayref([]);
248
            print "REINDEXING zebra\n";
241
        my @entries_to_update = map { $_->[0] } @$entries;
249
            print "====================\n";
242
250
        }
243
        C4::Search::IndexRecord($record_type, \@entries_to_update, {
251
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
244
            as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
252
        if ($process_zebraqueue) {
245
            noshadow => $noshadow, record_format => $record_fmt,
253
            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
246
            zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
254
                if %$records_deleted;
247
            verbose => $verbose_logging, skip_export => $skip_export,
255
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
248
            skip_index => $skip_index, keep_export => $keep_export,
256
                if $num_records_exported;
249
            directory => $directory
257
        } else {
250
        });
258
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
251
259
                if ($num_records_exported or $skip_export);
252
        unless ($do_not_clear_zebraqueue) {
253
            mark_all_zebraqueue_done($record_type);
260
        }
254
        }
261
    }
255
    }
262
}
256
}
Lines 326-647 sub select_all_biblios { Link Here
326
    return $sth;
320
    return $sth;
327
}
321
}
328
322
329
sub include_xml_wrapper {
330
    my $as_xml = shift;
331
    my $record_type = shift;
332
333
    return 0 unless $as_xml;
334
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
335
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
336
    return 0;
337
338
}
339
340
sub export_marc_records_from_sth {
341
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
342
343
    my $num_exported = 0;
344
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
345
    if (include_xml_wrapper($as_xml, $record_type)) {
346
        # include XML declaration and root element
347
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
348
    }
349
    my $i = 0;
350
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
351
    while (my ($record_number) = $sth->fetchrow_array) {
352
        print "." if ( $verbose_logging );
353
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
354
        if ( $nosanitize ) {
355
            my $marcxml = $record_type eq 'biblio'
356
                          ? GetXmlBiblio( $record_number )
357
                          : GetAuthorityXML( $record_number );
358
            if ($record_type eq 'biblio'){
359
                my @items = GetItemsInfo($record_number);
360
                if (@items){
361
                    my $record = MARC::Record->new;
362
                    $record->encoding('UTF-8');
363
                    my @itemsrecord;
364
                    foreach my $item (@items){
365
                        my $record = Item2Marc($item, $record_number);                        
366
                        push @itemsrecord, $record->field($itemtag);
367
                    }
368
                    $record->insert_fields_ordered(@itemsrecord);
369
                    my $itemsxml = $record->as_xml_record();
370
                    $marcxml =
371
                        substr($marcxml, 0, length($marcxml)-10) .
372
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
373
                }
374
            }
375
            if ( $marcxml ) {
376
                print {$fh} $marcxml if $marcxml;
377
                $num_exported++;
378
            }
379
            next;
380
        }
381
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
382
        if (defined $marc) {
383
            eval {
384
                my $rec;
385
                if ($as_xml) {
386
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
387
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
388
                } else {
389
                    $rec = $marc->as_usmarc();
390
                }
391
                print {$fh} $rec;
392
                $num_exported++;
393
            };
394
            if ($@) {
395
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
396
            }
397
        }
398
    }
399
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
400
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
401
    close $fh;
402
    return $num_exported;
403
}
404
405
sub export_marc_records_from_list {
406
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
407
408
    my $num_exported = 0;
409
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
410
    if (include_xml_wrapper($as_xml, $record_type)) {
411
        # include XML declaration and root element
412
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
413
    }
414
    my $i = 0;
415
416
    # Skip any deleted records. We check for this anyway, but this reduces error spam
417
    my %found = %$records_deleted;
418
    foreach my $record_number ( map { $_->{biblio_auth_number} }
419
                                grep { !$found{ $_->{biblio_auth_number} }++ }
420
                                @$entries ) {
421
        print "." if ( $verbose_logging );
422
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
423
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
424
        if (defined $marc) {
425
            eval {
426
                my $rec;
427
                if ($as_xml) {
428
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
429
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
430
                } else {
431
                    $rec = $marc->as_usmarc();
432
                }
433
                print {$fh} $rec;
434
                $num_exported++;
435
            };
436
            if ($@) {
437
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
438
            }
439
            $num_exported++;
440
        }
441
    }
442
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
443
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
444
    close $fh;
445
    return $num_exported;
446
}
447
448
sub generate_deleted_marc_records {
449
    my ($record_type, $entries, $directory, $as_xml) = @_;
450
451
    my $records_deleted = {};
452
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
453
    if (include_xml_wrapper($as_xml, $record_type)) {
454
        # include XML declaration and root element
455
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
456
    }
457
    my $i = 0;
458
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
459
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
460
        print "." if ( $verbose_logging );
461
462
        my $marc = MARC::Record->new();
463
        if ($record_type eq 'biblio') {
464
            fix_biblio_ids($marc, $record_number, $record_number);
465
        } else {
466
            fix_authority_id($marc, $record_number);
467
        }
468
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
469
            fix_unimarc_100($marc);
470
        }
471
472
        my $rec;
473
        if ($as_xml) {
474
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
475
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
476
        } else {
477
            $rec = $marc->as_usmarc();
478
        }
479
        print {$fh} $rec;
480
481
        $records_deleted->{$record_number} = 1;
482
    }
483
    print "\nRecords exported: $i\n" if ( $verbose_logging );
484
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
485
    close $fh;
486
    return $records_deleted;
487
    
488
489
}
490
491
sub get_corrected_marc_record {
492
    my ($record_type, $record_number, $noxml) = @_;
493
494
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml); 
495
496
    if (defined $marc) {
497
        fix_leader($marc);
498
        if ($record_type eq 'authority') {
499
            fix_authority_id($marc, $record_number);
500
        }
501
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
502
            fix_unimarc_100($marc);
503
        }
504
    }
505
506
    return $marc;
507
}
508
509
sub get_raw_marc_record {
510
    my ($record_type, $record_number, $noxml) = @_;
511
  
512
    my $marc; 
513
    if ($record_type eq 'biblio') {
514
        if ($noxml) {
515
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
516
            $fetch_sth->execute($record_number);
517
            if (my ($blob) = $fetch_sth->fetchrow_array) {
518
                $marc = MARC::Record->new_from_usmarc($blob);
519
                unless ($marc) {
520
                    warn "error creating MARC::Record from $blob";
521
                }
522
            }
523
            # failure to find a bib is not a problem -
524
            # a delete could have been done before
525
            # trying to process a record update
526
527
            $fetch_sth->finish();
528
            return unless $marc;
529
        } else {
530
            eval { $marc = GetMarcBiblio($record_number, 1); };
531
            if ($@ || !$marc) {
532
                # here we do warn since catching an exception
533
                # means that the bib was found but failed
534
                # to be parsed
535
                warn "error retrieving biblio $record_number";
536
                return;
537
            }
538
        }
539
    } else {
540
        eval { $marc = GetAuthority($record_number); };
541
        if ($@) {
542
            warn "error retrieving authority $record_number";
543
            return;
544
        }
545
    }
546
    return $marc;
547
}
548
549
sub fix_leader {
550
    # FIXME - this routine is suspect
551
    # It blanks the Leader/00-05 and Leader/12-16 to
552
    # force them to be recalculated correct when
553
    # the $marc->as_usmarc() or $marc->as_xml() is called.
554
    # But why is this necessary?  It would be a serious bug
555
    # in MARC::Record (definitely) and MARC::File::XML (arguably) 
556
    # if they are emitting incorrect leader values.
557
    my $marc = shift;
558
559
    my $leader = $marc->leader;
560
    substr($leader,  0, 5) = '     ';
561
    substr($leader, 10, 7) = '22     ';
562
    $marc->leader(substr($leader, 0, 24));
563
}
564
565
sub fix_biblio_ids {
566
    # FIXME - it is essential to ensure that the biblionumber is present,
567
    #         otherwise, Zebra will choke on the record.  However, this
568
    #         logic belongs in the relevant C4::Biblio APIs.
569
    my $marc = shift;
570
    my $biblionumber = shift;
571
    my $biblioitemnumber;
572
    if (@_) {
573
        $biblioitemnumber = shift;
574
    } else {    
575
        my $sth = $dbh->prepare(
576
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
577
        $sth->execute($biblionumber);
578
        ($biblioitemnumber) = $sth->fetchrow_array;
579
        $sth->finish;
580
        unless ($biblioitemnumber) {
581
            warn "failed to get biblioitemnumber for biblio $biblionumber";
582
            return 0;
583
        }
584
    }
585
586
    # FIXME - this is cheating on two levels
587
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
588
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
589
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
590
    #
591
    # On the other hand, this better for now than what rebuild_zebra.pl used to
592
    # do, which was duplicate the code for inserting the biblionumber 
593
    # and biblioitemnumber
594
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
595
596
    return 1;
597
}
598
599
sub fix_authority_id {
600
    # FIXME - as with fix_biblio_ids, the authid must be present
601
    #         for Zebra's sake.  However, this really belongs
602
    #         in C4::AuthoritiesMarc.
603
    my ($marc, $authid) = @_;
604
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
605
        $marc->delete_field($marc->field('001'));
606
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
607
    }
608
}
609
610
sub fix_unimarc_100 {
611
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
612
    my $marc = shift;
613
614
    my $string;
615
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
616
        $string = $marc->subfield( 100, "a" );
617
        my $f100 = $marc->field(100);
618
        $marc->delete_field($f100);
619
    }
620
    else {
621
        $string = POSIX::strftime( "%Y%m%d", localtime );
622
        $string =~ s/\-//g;
623
        $string = sprintf( "%-*s", 35, $string );
624
    }
625
    substr( $string, 22, 6, "frey50" );
626
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
627
        $marc->delete_field($marc->field(100));
628
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
629
    }
630
}
631
632
sub do_indexing {
633
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
634
635
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
636
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
637
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
638
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
639
640
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
641
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
642
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
643
644
}
645
323
646
sub print_usage {
324
sub print_usage {
647
    print <<_USAGE_;
325
    print <<_USAGE_;
648
- 

Return to bug 7419