View | Details | Raw Unified | Return to bug 7419
Collapse All | Expand All

(-)a/C4/Search.pm (+367 lines)
Lines 34-39 use C4::Charset; Link Here
34
use YAML;
34
use YAML;
35
use URI::Escape;
35
use URI::Escape;
36
use Business::ISBN;
36
use Business::ISBN;
37
use File::Temp qw/tempdir/;
37
38
38
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
39
use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG);
39
40
Lines 2712-2717 sub GetDistinctValues { Link Here
2712
   }
2713
   }
2713
}
2714
}
2714
2715
2716
sub IndexRecord {
2717
    my ($recordtype, $recordids, $options) = @_;
2718
2719
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
2720
    my $noxml = ($options and $options->{noxml}) ? 1 : 0;
2721
    my $nosanitize = ($options and $options->{nosanitize}) ? $as_xml : 0;
2722
    my $reset_index = ($options and $options->{reset_index}) ? 1 : 0;
2723
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
2724
    my $record_format = ($options and $options->{record_format})
2725
                    ? $options->{record_format}
2726
                    : ($as_xml)
2727
                        ? 'marcxml'
2728
                        : 'iso2709';
2729
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
2730
                        ? $options->{zebraidx_log_opt}
2731
                        : "";
2732
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
2733
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
2734
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
2735
    my $directory = ($options and $options->{directory})
2736
                    ? $options->{directory}
2737
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
2738
2739
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
2740
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
2741
2742
    my $num_exported = 0;
2743
    unless ($skip_export) {
2744
        $num_exported = _export_marc_records_from_list($recordtype, \@recordids,
2745
            $directory, $as_xml, $noxml, $nosanitize, $verbose);
2746
    }
2747
    _do_indexing($recordtype, "update", $directory, $reset_index, $noshadow,
2748
        $record_format, $zebraidx_log_opt);
2749
2750
    return $num_exported;
2751
}
2752
2753
sub DeleteRecordIndex {
2754
    my ($recordtype, $recordids, $options) = @_;
2755
2756
    my $as_xml = ($options and $options->{as_xml}) ? 1 : 0;
2757
    my $noshadow = ($options and $options->{noshadow}) ? " -n " : "";
2758
    my $record_format = ($options and $options->{record_format})
2759
                    ? $options->{record_format}
2760
                    : ($as_xml)
2761
                        ? 'marcxml'
2762
                        : 'iso2709';
2763
    my $zebraidx_log_opt = ($options and defined $options->{zebraidx_log_opt})
2764
                        ? $options->{zebraidx_log_opt}
2765
                        : "";
2766
    my $verbose = ($options and $options->{verbose}) ? 1 : 0;
2767
    my $skip_export = ($options and $options->{skip_export}) ? 1 : 0;
2768
    my $keep_export = ($options and $options->{keep_export}) ? 1 : 0;
2769
    my $directory = ($options and $options->{directory})
2770
                    ? $options->{directory}
2771
                    : tempdir(CLEANUP => ($keep_export ? 0 : 1));
2772
2773
    my @recordids = (ref $recordids eq "ARRAY") ? @$recordids : split (/ /, $recordids);
2774
    @recordids = map { { biblio_auth_number => $_ } } @recordids;
2775
2776
    my $num_exported = 0;
2777
    unless ($skip_export) {
2778
        $num_exported = _generate_deleted_marc_records($recordtype, \@recordids,
2779
            $directory, $as_xml, $verbose);
2780
    }
2781
    _do_indexing($recordtype, "delete", $directory, 0, $noshadow,
2782
        $record_format, $zebraidx_log_opt);
2783
2784
    return $num_exported;
2785
}
2786
2787
2788
sub _export_marc_records_from_list {
2789
    my ( $record_type, $entries, $directory, $as_xml, $noxml, $nosanitize, $verbose ) = @_;
2790
2791
    my $num_exported = 0;
2792
    open my $fh, ">:encoding(UTF-8) ", "$directory/exported_records" or die $!;
2793
    if (_include_xml_wrapper($as_xml, $record_type)) {
2794
        # include XML declaration and root element
2795
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
2796
    }
2797
    my $i     = 0;
2798
    my %found = ();
2799
    my ($itemtag) = GetMarcFromKohaField("items.itemnumber",'');
2800
    foreach my $record_number (
2801
        map  { $_->{biblio_auth_number} }
2802
        grep { !$found{ $_->{biblio_auth_number} }++ } @$entries
2803
    ) {
2804
        if($nosanitize) {
2805
            my $marcxml = $record_type eq 'biblio'
2806
                        ? GetXmlBiblio($record_number)
2807
                        : GetAuthorityXML($record_number);
2808
            if ($record_type eq 'biblio'){
2809
                my @items = GetItemsInfo($record_number);
2810
                if (@items){
2811
                    my $record = MARC::Record->new;
2812
                    $record->encoding('UTF-8');
2813
                    my @itemsrecord;
2814
                    foreach my $item (@items){
2815
                        my $record = Item2Marc($item, $record_number);
2816
                        push @itemsrecord, $record->field($itemtag);
2817
                    }
2818
                    $record->insert_fields_ordered(@itemsrecord);
2819
                    my $itemsxml = $record->as_xml_record();
2820
                    $marcxml =
2821
                        substr($marcxml, 0, length($marcxml)-10) .
2822
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
2823
                }
2824
            }
2825
            if ($marcxml) {
2826
                print {$fh} $marcxml;
2827
                $num_exported++;
2828
            }
2829
        } else {
2830
            my ($marc) = _get_corrected_marc_record( $record_type, $record_number, $noxml );
2831
            if ( defined $marc ) {
2832
                eval {
2833
                    my $rec;
2834
                    if ($as_xml) {
2835
                        $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
2836
                        $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
2837
                    } else {
2838
                        $rec = $marc->as_usmarc();
2839
                    }
2840
                    print {$fh} $rec;
2841
                    $num_exported++;
2842
                };
2843
                if ($@) {
2844
                    warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
2845
                }
2846
            }
2847
        }
2848
        $i++;
2849
        if($verbose) {
2850
            print ".";
2851
            print "$i\n" unless($i % 100);
2852
        }
2853
    }
2854
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
2855
    close $fh;
2856
    print "\n$num_exported records exported to $directory/exported_records\n" if $verbose;
2857
    return $num_exported;
2858
}
2859
2860
sub _generate_deleted_marc_records {
2861
    my ( $record_type, $entries, $directory, $as_xml, $verbose ) = @_;
2862
2863
    my $num_exported = 0;
2864
    open my $fh, ">:encoding(UTF-8)", "$directory/exported_records" or die $!;
2865
    if (_include_xml_wrapper($as_xml, $record_type)) {
2866
        # include XML declaration and root element
2867
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
2868
    }
2869
    my $i = 0;
2870
    foreach my $record_number ( map { $_->{biblio_auth_number} } @$entries ) {
2871
        my $marc = MARC::Record->new();
2872
        if ( $record_type eq 'biblio' ) {
2873
            _fix_biblio_ids( $marc, $record_number, $record_number );
2874
        } else {
2875
            _fix_authority_id( $marc, $record_number );
2876
        }
2877
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
2878
            _fix_unimarc_100($marc);
2879
        }
2880
2881
        my $rec;
2882
        if ($as_xml) {
2883
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
2884
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
2885
        } else {
2886
            $rec = $marc->as_usmarc();
2887
        }
2888
        print {$fh} $rec;
2889
2890
        $num_exported++;
2891
        $i++;
2892
        if($verbose) {
2893
            print ".";
2894
            print "$i\n" unless($i % 100);
2895
        }
2896
    }
2897
    print {$fh} '</collection>' if (_include_xml_wrapper($as_xml, $record_type));
2898
    close $fh;
2899
2900
    return $num_exported;
2901
}
2902
2903
sub _get_corrected_marc_record {
2904
    my ( $record_type, $record_number, $noxml ) = @_;
2905
2906
    my $marc = _get_raw_marc_record( $record_type, $record_number, $noxml );
2907
2908
    if ( defined $marc ) {
2909
        _fix_leader($marc);
2910
        if ( $record_type eq 'biblio' ) {
2911
            my $succeeded = _fix_biblio_ids( $marc, $record_number );
2912
            return unless $succeeded;
2913
        } else {
2914
            _fix_authority_id( $marc, $record_number );
2915
        }
2916
        if ( C4::Context->preference("marcflavour") eq "UNIMARC" ) {
2917
            _fix_unimarc_100($marc);
2918
        }
2919
    }
2920
2921
    return $marc;
2922
}
2923
2924
sub _get_raw_marc_record {
2925
    my ( $record_type, $record_number, $noxml ) = @_;
2926
2927
    my $dbh = C4::Context->dbh;
2928
    my $marc;
2929
    if ( $record_type eq 'biblio' ) {
2930
        if ($noxml) {
2931
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems
2932
                WHERE biblionumber = ?");
2933
            $fetch_sth->execute($record_number);
2934
            if ( my ($blob) = $fetch_sth->fetchrow_array ) {
2935
                $marc = MARC::Record->new_from_usmarc($blob);
2936
                unless ($marc) {
2937
                    warn "error creating MARC::Record from $blob";
2938
                }
2939
            }
2940
            # failure to find a bib is not a problem -
2941
            # a delete could have been done before
2942
            # trying to process a record update
2943
2944
            $fetch_sth->finish();
2945
            return unless $marc;
2946
        } else {
2947
            eval { $marc = GetMarcBiblio($record_number, 1); };
2948
            if ($@ || !$marc) {
2949
                # here we do warn since catching an exception
2950
                # means that the bib was found but failed
2951
                # to be parsed
2952
                warn "error retrieving biblio $record_number: $@";
2953
                return;
2954
            }
2955
        }
2956
    } else {
2957
        eval { $marc = C4::AuthoritiesMarc::GetAuthority($record_number); };
2958
        if ($@) {
2959
            warn "error retrieving authority $record_number: $@";
2960
            return;
2961
        }
2962
    }
2963
    return $marc;
2964
}
2965
2966
sub _fix_leader {
2967
2968
    # FIXME - this routine is suspect
2969
    # It blanks the Leader/00-05 and Leader/12-16 to
2970
    # force them to be recalculated correct when
2971
    # the $marc->as_usmarc() or $marc->as_xml() is called.
2972
    # But why is this necessary?  It would be a serious bug
2973
    # in MARC::Record (definitely) and MARC::File::XML (arguably)
2974
    # if they are emitting incorrect leader values.
2975
    my $marc = shift;
2976
2977
    my $leader = $marc->leader;
2978
    substr( $leader, 0,  5 ) = '     ';
2979
    substr( $leader, 10, 7 ) = '22     ';
2980
    $marc->leader( substr( $leader, 0, 24 ) );
2981
}
2982
2983
2984
sub _fix_biblio_ids {
2985
2986
    # FIXME - it is essential to ensure that the biblionumber is present,
2987
    #         otherwise, Zebra will choke on the record.  However, this
2988
    #         logic belongs in the relevant C4::Biblio APIs.
2989
    my $marc         = shift;
2990
    my $biblionumber = shift;
2991
    my $dbh = C4::Context->dbh;
2992
    my $biblioitemnumber;
2993
    if (@_) {
2994
        $biblioitemnumber = shift;
2995
    } else {
2996
        my $sth = $dbh->prepare(
2997
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
2998
        $sth->execute($biblionumber);
2999
        ($biblioitemnumber) = $sth->fetchrow_array;
3000
        $sth->finish;
3001
        unless ($biblioitemnumber) {
3002
            warn "failed to get biblioitemnumber for biblio $biblionumber";
3003
            return 0;
3004
        }
3005
    }
3006
3007
    # FIXME - this is cheating on two levels
3008
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
3009
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
3010
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
3011
    #
3012
    # On the other hand, this better for now than what rebuild_zebra.pl used to
3013
    # do, which was duplicate the code for inserting the biblionumber
3014
    # and biblioitemnumber
3015
    C4::Biblio::_koha_marc_update_bib_ids( $marc, '', $biblionumber, $biblioitemnumber );
3016
3017
    return 1;
3018
}
3019
3020
sub _fix_authority_id {
3021
3022
    # FIXME - as with fix_biblio_ids, the authid must be present
3023
    #         for Zebra's sake.  However, this really belongs
3024
    #         in C4::AuthoritiesMarc.
3025
    my ( $marc, $authid ) = @_;
3026
    unless ( $marc->field('001') and $marc->field('001')->data() eq $authid ) {
3027
        $marc->delete_field( $marc->field('001') );
3028
        $marc->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
3029
    }
3030
}
3031
3032
sub _fix_unimarc_100 {
3033
3034
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
3035
    my $marc = shift;
3036
3037
    my $string;
3038
    if ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
3039
        $string = $marc->subfield( 100, "a" );
3040
        my $f100 = $marc->field(100);
3041
        $marc->delete_field($f100);
3042
    } else {
3043
        $string = POSIX::strftime( "%Y%m%d", localtime );
3044
        $string =~ s/\-//g;
3045
        $string = sprintf( "%-*s", 35, $string );
3046
    }
3047
    substr( $string, 22, 6, "frey50" );
3048
    unless ( length( $marc->subfield( 100, "a" ) ) == 36 ) {
3049
        $marc->delete_field( $marc->field(100) );
3050
        $marc->insert_grouped_field( MARC::Field->new( 100, "", "", "a" => $string ) );
3051
    }
3052
}
3053
3054
sub _do_indexing {
3055
    my ( $record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt ) = @_;
3056
3057
    my $zebra_server  = ( $record_type eq 'biblio' ) ? 'biblioserver' : 'authorityserver';
3058
    my $zebra_db_name = ( $record_type eq 'biblio' ) ? 'biblios'      : 'authorities';
3059
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
3060
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
3061
3062
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
3063
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
3064
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
3065
3066
}
3067
3068
sub _include_xml_wrapper {
3069
    my $as_xml = shift;
3070
    my $record_type = shift;
3071
3072
    my $bib_index_mode = C4::Context->config('zebra_bib_index_mode') || 'grs1';
3073
    my $auth_index_mode = C4::Context->config('zebra_auth_index_mode') || 'dom';
3074
3075
    return 0 unless $as_xml;
3076
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
3077
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
3078
    return 0;
3079
3080
}
3081
2715
3082
2716
END { }    # module clean-up code here (global destructor)
3083
END { }    # module clean-up code here (global destructor)
2717
3084
(-)a/misc/migration_tools/rebuild_zebra.pl (-359 / +47 lines)
Lines 10-15 use File::Path; Link Here
10
use C4::Biblio;
10
use C4::Biblio;
11
use C4::AuthoritiesMarc;
11
use C4::AuthoritiesMarc;
12
use C4::Items;
12
use C4::Items;
13
use File::Temp qw(tempdir);
13
14
14
# 
15
# 
15
# script that checks zebradir structure & create directories & mandatory files if needed
16
# script that checks zebradir structure & create directories & mandatory files if needed
Lines 155-166 if ( $verbose_logging ) { Link Here
155
}
156
}
156
if ($keep_export) {
157
if ($keep_export) {
157
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "NOTHING cleaned : the export $directory has been kept.\n";
158
    print "You can re-run this script with the -s ";
159
    print "You can re-run this script with the -s and -d $directory parameters";
159
    if ($use_tempdir) {
160
        print " and -d $directory parameters";
161
    } else {
162
        print "parameter";
163
    }
164
    print "\n";
160
    print "\n";
165
    print "if you just want to rebuild zebra after changing the record.abs\n";
161
    print "if you just want to rebuild zebra after changing the record.abs\n";
166
    print "or another zebra config file\n";
162
    print "or another zebra config file\n";
Lines 195-202 sub check_zebra_dirs { Link Here
195
sub index_records {
191
sub index_records {
196
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
192
    my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
197
193
198
    my $num_records_exported = 0;
199
    my $records_deleted;
200
    my $need_reset = check_zebra_dirs($server_dir);
194
    my $need_reset = check_zebra_dirs($server_dir);
201
    if ($need_reset) {
195
    if ($need_reset) {
202
    	print "$0: found broken zebra server directories: forcing a rebuild\n";
196
    	print "$0: found broken zebra server directories: forcing a rebuild\n";
Lines 206-241 sub index_records { Link Here
206
        print "====================\n";
200
        print "====================\n";
207
        print "SKIPPING $record_type export\n";
201
        print "SKIPPING $record_type export\n";
208
        print "====================\n";
202
        print "====================\n";
209
    } else {
210
        if ( $verbose_logging ) {
211
            print "====================\n";
212
            print "exporting $record_type\n";
213
            print "====================\n";
214
        }
215
        mkdir "$directory" unless (-d $directory);
216
        mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
217
        if ($process_zebraqueue) {
218
            my $entries = select_zebraqueue_records($record_type, 'deleted');
219
            mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
220
            $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
221
            mark_zebraqueue_batch_done($entries);
222
            $entries = select_zebraqueue_records($record_type, 'updated');
223
            mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
224
            $num_records_exported = export_marc_records_from_list($record_type, 
225
                                                                  $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
226
            mark_zebraqueue_batch_done($entries);
227
        } else {
228
            my $sth = select_all_records($record_type);
229
            $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
230
            unless ($do_not_clear_zebraqueue) {
231
                mark_all_zebraqueue_done($record_type);
232
            }
233
        }
234
    }
203
    }
235
204
236
    #
237
    # and reindexing everything
238
    #
239
    if ($skip_index) {
205
    if ($skip_index) {
240
        if ($verbose_logging) {
206
        if ($verbose_logging) {
241
            print "====================\n";
207
            print "====================\n";
Lines 250-262 sub index_records { Link Here
250
        }
216
        }
251
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
217
        my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
252
        if ($process_zebraqueue) {
218
        if ($process_zebraqueue) {
253
            do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
219
            # Process 'deleted' records
254
                if %$records_deleted;
220
            my $entries = select_zebraqueue_records( $record_type, 'deleted' );
255
            do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
221
            my @entries_to_delete = map { $_->{biblio_auth_id} } @$entries;
256
                if $num_records_exported;
222
223
            C4::Search::DeleteRecordIndex($record_type, \@entries_to_delete, {
224
                as_xml => $as_xml, noshadow => $noshadow, record_format => $record_fmt,
225
                zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
226
                skip_export => $skip_export, keep_export => $keep_export,
227
                directory => $directory
228
            });
229
230
            mark_zebraqueue_batch_done($entries);
231
232
            # Process 'updated' records'
233
            $entries = select_zebraqueue_records( $record_type, 'updated' );
234
            my @entries_to_update = map {
235
                my $id = $_->{biblio_auth_id};
236
                # do not try to update deleted records
237
                (0 == grep { $id == $_ } @entries_to_delete) ? $id : ()
238
            } @$entries;
239
240
            C4::Search::IndexRecord($record_type, \@entries_to_update, {
241
                as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
242
                noshadow => $noshadow, record_format => $record_fmt,
243
                zebraidx_log_opt => $zebraidx_log_opt, verbose => $verbose_logging,
244
                skip_export => $skip_export, keep_export => $keep_export,
245
                directory => $directory
246
            });
247
248
            mark_zebraqueue_batch_done($entries);
257
        } else {
249
        } else {
258
            do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt)
250
            my $sth = select_all_records($record_type);
259
                if ($num_records_exported or $skip_export);
251
            my $entries = $sth->fetchall_arrayref([]);
252
            my @entries_to_update = map { $_->[0] } @$entries;
253
254
            C4::Search::IndexRecord($record_type, \@entries_to_update, {
255
                as_xml => $as_xml, noxml => $noxml, reset_index => $reset,
256
                noshadow => $noshadow, record_format => $record_fmt,
257
                zebraidx_log_opt => $zebraidx_log_opt, nosanitize => $nosanitize,
258
                verbose => $verbose_logging, skip_export => $skip_export,
259
                keep_export => $keep_export, directory => $directory
260
            });
261
262
            unless ($do_not_clear_zebraqueue) {
263
                mark_all_zebraqueue_done($record_type);
264
            }
260
        }
265
        }
261
    }
266
    }
262
}
267
}
Lines 326-647 sub select_all_biblios { Link Here
326
    return $sth;
331
    return $sth;
327
}
332
}
328
333
329
sub include_xml_wrapper {
330
    my $as_xml = shift;
331
    my $record_type = shift;
332
333
    return 0 unless $as_xml;
334
    return 1 if $record_type eq 'biblio' and $bib_index_mode eq 'dom';
335
    return 1 if $record_type eq 'authority' and $auth_index_mode eq 'dom';
336
    return 0;
337
338
}
339
340
sub export_marc_records_from_sth {
341
    my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_;
342
343
    my $num_exported = 0;
344
    open my $fh, '>:encoding(UTF-8) ', "$directory/exported_records" or die $!;
345
    if (include_xml_wrapper($as_xml, $record_type)) {
346
        # include XML declaration and root element
347
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
348
    }
349
    my $i = 0;
350
    my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField("items.itemnumber",'');
351
    while (my ($record_number) = $sth->fetchrow_array) {
352
        print "." if ( $verbose_logging );
353
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
354
        if ( $nosanitize ) {
355
            my $marcxml = $record_type eq 'biblio'
356
                          ? GetXmlBiblio( $record_number )
357
                          : GetAuthorityXML( $record_number );
358
            if ($record_type eq 'biblio'){
359
                my @items = GetItemsInfo($record_number);
360
                if (@items){
361
                    my $record = MARC::Record->new;
362
                    $record->encoding('UTF-8');
363
                    my @itemsrecord;
364
                    foreach my $item (@items){
365
                        my $record = Item2Marc($item, $record_number);                        
366
                        push @itemsrecord, $record->field($itemtag);
367
                    }
368
                    $record->insert_fields_ordered(@itemsrecord);
369
                    my $itemsxml = $record->as_xml_record();
370
                    $marcxml =
371
                        substr($marcxml, 0, length($marcxml)-10) .
372
                        substr($itemsxml, index($itemsxml, "</leader>\n", 0) + 10);
373
                }
374
            }
375
            if ( $marcxml ) {
376
                print {$fh} $marcxml if $marcxml;
377
                $num_exported++;
378
            }
379
            next;
380
        }
381
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
382
        if (defined $marc) {
383
            eval {
384
                my $rec;
385
                if ($as_xml) {
386
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
387
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
388
                } else {
389
                    $rec = $marc->as_usmarc();
390
                }
391
                print {$fh} $rec;
392
                $num_exported++;
393
            };
394
            if ($@) {
395
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
396
            }
397
        }
398
    }
399
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
400
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
401
    close $fh;
402
    return $num_exported;
403
}
404
405
sub export_marc_records_from_list {
406
    my ($record_type, $entries, $directory, $as_xml, $noxml, $records_deleted) = @_;
407
408
    my $num_exported = 0;
409
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
410
    if (include_xml_wrapper($as_xml, $record_type)) {
411
        # include XML declaration and root element
412
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
413
    }
414
    my $i = 0;
415
416
    # Skip any deleted records. We check for this anyway, but this reduces error spam
417
    my %found = %$records_deleted;
418
    foreach my $record_number ( map { $_->{biblio_auth_number} }
419
                                grep { !$found{ $_->{biblio_auth_number} }++ }
420
                                @$entries ) {
421
        print "." if ( $verbose_logging );
422
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
423
        my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
424
        if (defined $marc) {
425
            eval {
426
                my $rec;
427
                if ($as_xml) {
428
                    $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
429
                    $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
430
                } else {
431
                    $rec = $marc->as_usmarc();
432
                }
433
                print {$fh} $rec;
434
                $num_exported++;
435
            };
436
            if ($@) {
437
              warn "Error exporting record $record_number ($record_type) ".($noxml ? "not XML" : "XML");
438
            }
439
            $num_exported++;
440
        }
441
    }
442
    print "\nRecords exported: $num_exported\n" if ( $verbose_logging );
443
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
444
    close $fh;
445
    return $num_exported;
446
}
447
448
sub generate_deleted_marc_records {
449
    my ($record_type, $entries, $directory, $as_xml) = @_;
450
451
    my $records_deleted = {};
452
    open my $fh, '>:encoding(UTF-8)', "$directory/exported_records" or die $!;
453
    if (include_xml_wrapper($as_xml, $record_type)) {
454
        # include XML declaration and root element
455
        print {$fh} '<?xml version="1.0" encoding="UTF-8"?><collection>';
456
    }
457
    my $i = 0;
458
    foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) {
459
        print "\r$i" unless ($i++ %100 or !$verbose_logging);
460
        print "." if ( $verbose_logging );
461
462
        my $marc = MARC::Record->new();
463
        if ($record_type eq 'biblio') {
464
            fix_biblio_ids($marc, $record_number, $record_number);
465
        } else {
466
            fix_authority_id($marc, $record_number);
467
        }
468
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
469
            fix_unimarc_100($marc);
470
        }
471
472
        my $rec;
473
        if ($as_xml) {
474
            $rec = $marc->as_xml_record(C4::Context->preference('marcflavour'));
475
            $rec =~ s!<\?xml version="1.0" encoding="UTF-8"\?>\n!!;
476
        } else {
477
            $rec = $marc->as_usmarc();
478
        }
479
        print {$fh} $rec;
480
481
        $records_deleted->{$record_number} = 1;
482
    }
483
    print "\nRecords exported: $i\n" if ( $verbose_logging );
484
    print {$fh} '</collection>' if (include_xml_wrapper($as_xml, $record_type));
485
    close $fh;
486
    return $records_deleted;
487
    
488
489
}
490
491
sub get_corrected_marc_record {
492
    my ($record_type, $record_number, $noxml) = @_;
493
494
    my $marc = get_raw_marc_record($record_type, $record_number, $noxml); 
495
496
    if (defined $marc) {
497
        fix_leader($marc);
498
        if ($record_type eq 'authority') {
499
            fix_authority_id($marc, $record_number);
500
        }
501
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
502
            fix_unimarc_100($marc);
503
        }
504
    }
505
506
    return $marc;
507
}
508
509
sub get_raw_marc_record {
510
    my ($record_type, $record_number, $noxml) = @_;
511
  
512
    my $marc; 
513
    if ($record_type eq 'biblio') {
514
        if ($noxml) {
515
            my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
516
            $fetch_sth->execute($record_number);
517
            if (my ($blob) = $fetch_sth->fetchrow_array) {
518
                $marc = MARC::Record->new_from_usmarc($blob);
519
                unless ($marc) {
520
                    warn "error creating MARC::Record from $blob";
521
                }
522
            }
523
            # failure to find a bib is not a problem -
524
            # a delete could have been done before
525
            # trying to process a record update
526
527
            $fetch_sth->finish();
528
            return unless $marc;
529
        } else {
530
            eval { $marc = GetMarcBiblio($record_number, 1); };
531
            if ($@ || !$marc) {
532
                # here we do warn since catching an exception
533
                # means that the bib was found but failed
534
                # to be parsed
535
                warn "error retrieving biblio $record_number";
536
                return;
537
            }
538
        }
539
    } else {
540
        eval { $marc = GetAuthority($record_number); };
541
        if ($@) {
542
            warn "error retrieving authority $record_number";
543
            return;
544
        }
545
    }
546
    return $marc;
547
}
548
549
sub fix_leader {
550
    # FIXME - this routine is suspect
551
    # It blanks the Leader/00-05 and Leader/12-16 to
552
    # force them to be recalculated correct when
553
    # the $marc->as_usmarc() or $marc->as_xml() is called.
554
    # But why is this necessary?  It would be a serious bug
555
    # in MARC::Record (definitely) and MARC::File::XML (arguably) 
556
    # if they are emitting incorrect leader values.
557
    my $marc = shift;
558
559
    my $leader = $marc->leader;
560
    substr($leader,  0, 5) = '     ';
561
    substr($leader, 10, 7) = '22     ';
562
    $marc->leader(substr($leader, 0, 24));
563
}
564
565
sub fix_biblio_ids {
566
    # FIXME - it is essential to ensure that the biblionumber is present,
567
    #         otherwise, Zebra will choke on the record.  However, this
568
    #         logic belongs in the relevant C4::Biblio APIs.
569
    my $marc = shift;
570
    my $biblionumber = shift;
571
    my $biblioitemnumber;
572
    if (@_) {
573
        $biblioitemnumber = shift;
574
    } else {    
575
        my $sth = $dbh->prepare(
576
            "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
577
        $sth->execute($biblionumber);
578
        ($biblioitemnumber) = $sth->fetchrow_array;
579
        $sth->finish;
580
        unless ($biblioitemnumber) {
581
            warn "failed to get biblioitemnumber for biblio $biblionumber";
582
            return 0;
583
        }
584
    }
585
586
    # FIXME - this is cheating on two levels
587
    # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
588
    # 2. Making sure that the biblionumber and biblioitemnumber are correct and
589
    #    present in the MARC::Record object ought to be part of GetMarcBiblio.
590
    #
591
    # On the other hand, this better for now than what rebuild_zebra.pl used to
592
    # do, which was duplicate the code for inserting the biblionumber 
593
    # and biblioitemnumber
594
    C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
595
596
    return 1;
597
}
598
599
sub fix_authority_id {
600
    # FIXME - as with fix_biblio_ids, the authid must be present
601
    #         for Zebra's sake.  However, this really belongs
602
    #         in C4::AuthoritiesMarc.
603
    my ($marc, $authid) = @_;
604
    unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
605
        $marc->delete_field($marc->field('001'));
606
        $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
607
    }
608
}
609
610
sub fix_unimarc_100 {
611
    # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
612
    my $marc = shift;
613
614
    my $string;
615
    if ( length($marc->subfield( 100, "a" )) == 36 ) {
616
        $string = $marc->subfield( 100, "a" );
617
        my $f100 = $marc->field(100);
618
        $marc->delete_field($f100);
619
    }
620
    else {
621
        $string = POSIX::strftime( "%Y%m%d", localtime );
622
        $string =~ s/\-//g;
623
        $string = sprintf( "%-*s", 35, $string );
624
    }
625
    substr( $string, 22, 6, "frey50" );
626
    unless ( length($marc->subfield( 100, "a" )) == 36 ) {
627
        $marc->delete_field($marc->field(100));
628
        $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
629
    }
630
}
631
632
sub do_indexing {
633
    my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_;
634
635
    my $zebra_server  = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
636
    my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities';
637
    my $zebra_config  = C4::Context->zebraconfig($zebra_server)->{'config'};
638
    my $zebra_db_dir  = C4::Context->zebraconfig($zebra_server)->{'directory'};
639
640
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index;
641
    system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir");
642
    system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow;
643
644
}
645
334
646
sub print_usage {
335
sub print_usage {
647
    print <<_USAGE_;
336
    print <<_USAGE_;
648
- 

Return to bug 7419