View | Details | Raw Unified | Return to bug 20551
Collapse All | Expand All

(-)a/Koha/Exporter/Record.pm (-29 / +101 lines)
Lines 6-18 use MARC::File::USMARC; Link Here
6
6
7
use C4::AuthoritiesMarc;
7
use C4::AuthoritiesMarc;
8
use C4::Biblio qw( GetMarcFromKohaField );
8
use C4::Biblio qw( GetMarcFromKohaField );
9
use C4::Charset;
9
use C4::Record;
10
use C4::Record;
10
use Koha::Biblios;
11
use Koha::Biblios;
11
use Koha::CsvProfiles;
12
use Koha::CsvProfiles;
13
use Koha::Database;
12
use Koha::Logger;
14
use Koha::Logger;
13
use Koha::RecordProcessor;
15
use Koha::RecordProcessor;
14
use List::Util qw( all any );
16
use List::Util qw( all any );
15
17
18
use MARC::Record;
19
use MARC::File::XML;
20
16
sub _get_record_for_export {
21
sub _get_record_for_export {
17
    my ($params)           = @_;
22
    my ($params)           = @_;
18
    my $record_type        = $params->{record_type};
23
    my $record_type        = $params->{record_type};
Lines 105-110 sub _get_record_for_export { Link Here
105
    return $record;
110
    return $record;
106
}
111
}
107
112
113
sub _get_deleted_biblio_for_export {
114
    my ($params) = @_;
115
    my $biblionumber = $params->{biblionumber};
116
117
    # Creating schema is expensive, allow caller to
118
    # pass it so don't have to recreate for each call
119
    my $resultset       = $params->{resultset} || Koha::Database->new()->schema()->resultset('DeletedbiblioMetadata');
120
    my $marc_flavour    = C4::Context->preference('marcflavour');
121
    my $biblio_metadata = $resultset->find(
122
        {
123
            'biblionumber' => $biblionumber,
124
            'format'       => 'marcxml',
125
            'marcflavour'  => $marc_flavour
126
        }
127
    );
128
    my $marc_xml = $biblio_metadata->metadata;
129
    $marc_xml = StripNonXmlChars($marc_xml);
130
131
    my $record = eval { MARC::Record::new_from_xml( $marc_xml, 'UTF-8', $marc_flavour ) };
132
    if ( !$record ) {
133
        Koha::Logger->get->warn("Failed to load MARCXML for deleted biblio with biblionumber \"$biblionumber\": $@");
134
        return;
135
    }
136
137
    # Set deleted flag (record status, position 05)
138
    my $leader = $record->leader;
139
    substr $leader, 5, 1, 'd';
140
    $record->leader($leader);
141
    return $record;
142
}
143
108
sub _get_authority_for_export {
144
sub _get_authority_for_export {
109
    my ($params)  = @_;
145
    my ($params)  = @_;
110
    my $authid    = $params->{authid} || return;
146
    my $authid    = $params->{authid} || return;
Lines 124-130 sub _get_biblio_for_export { Link Here
124
    my $biblio = Koha::Biblios->find($biblionumber);
160
    my $biblio = Koha::Biblios->find($biblionumber);
125
    my $record = eval { $biblio->metadata->record };
161
    my $record = eval { $biblio->metadata->record };
126
162
127
    return if $@ or not defined $record;
163
    if ( !$record ) {
164
        Koha::Logger->get->warn("Failed to load MARCXML for biblio with biblionumber \"$biblionumber\": $@");
165
        return;
166
    }
128
167
129
    if ($embed_see_from_headings) {
168
    if ($embed_see_from_headings) {
130
        my $record_processor = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
169
        my $record_processor = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
Lines 159-165 sub export { Link Here
159
    my ($params) = @_;
198
    my ($params) = @_;
160
199
161
    my $record_type        = $params->{record_type};
200
    my $record_type        = $params->{record_type};
162
    my $record_ids         = $params->{record_ids} || [];
201
    my $record_ids         = $params->{record_ids}         || [];
202
    my $deleted_record_ids = $params->{deleted_record_ids} || [];
163
    my $format             = $params->{format};
203
    my $format             = $params->{format};
164
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
204
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
165
    my $export_items       = $params->{export_items};
205
    my $export_items       = $params->{export_items};
Lines 171-177 sub export { Link Here
171
        Koha::Logger->get->warn("No record_type given.");
211
        Koha::Logger->get->warn("No record_type given.");
172
        return;
212
        return;
173
    }
213
    }
174
    return unless @$record_ids;
214
    return unless ( @{$record_ids} || @{$deleted_record_ids} && $format ne 'csv' );
175
215
176
    my $fh;
216
    my $fh;
177
    if ($output_filepath) {
217
    if ($output_filepath) {
Lines 182-221 sub export { Link Here
182
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
222
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
183
    }
223
    }
184
224
185
    if ( $format eq 'iso2709' ) {
225
    if ( $format eq 'xml' || $format eq 'iso2709' ) {
186
        for my $record_id (@$record_ids) {
226
        my @records;
187
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
227
        @records = map {
188
            next unless $record;
228
            my $record = _get_record_for_export( { %{$params}, record_id => $_ } );
189
            my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
229
            $record ? $record : ();
190
            if ( $errorcount_on_decode or $@ ) {
230
        } @{$record_ids};
191
                my $msg = "Record $record_id could not be exported. " . ( $@ // '' );
231
192
                chomp $msg;
232
        my @deleted_records;
193
                Koha::Logger->get->info($msg);
233
        if ( @{$deleted_record_ids} ) {
194
                next;
234
            my $resultset = Koha::Database->new()->schema()->resultset('DeletedbiblioMetadata');
195
            }
235
            @deleted_records = map {
196
            print $record->as_usmarc();
236
                my $record = _get_deleted_biblio_for_export(
237
                    {
238
                        biblionumber => $_,
239
                        resultset    => $resultset,
240
                    }
241
                );
242
                $record ? $record : ();
243
            } @{$deleted_record_ids};
197
        }
244
        }
198
    } elsif ( $format eq 'xml' ) {
245
        if ( $format eq 'iso2709' ) {
199
        my $marcflavour = C4::Context->preference("marcflavour");
246
            my $encoding_validator = sub {
200
        MARC::File::XML->default_record_format(
247
                my ($record_type) = @_;
201
            ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
248
                return sub {
202
249
                    my ($record) = @_;
203
        print MARC::File::XML::header();
250
                    my $errorcount_on_decode =
204
        print "\n";
251
                        eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
205
        for my $record_id (@$record_ids) {
252
                    if ( $errorcount_on_decode || $@ ) {
206
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
253
                        my ( $id_tag, $id_subfield ) = GetMarcFromKohaField( 'biblio.biblionumber', '' );
207
            next unless $record;
254
                        my $record_id = $record->subfield( $id_tag, $id_subfield );
208
            print MARC::File::XML::record($record);
255
                        my $msg       = "$record_type $record_id could not be USMARC decoded/encoded. " . ( $@ // '' );
256
                        chomp $msg;
257
                        Koha::Logger->get->warn($msg);
258
                        return 0;
259
                    }
260
                    return 1;
261
                }
262
            };
263
            my $validator = $encoding_validator->('Record');
264
            for my $record ( grep { $validator->($_) } @records ) {
265
                print $record->as_usmarc();
266
            }
267
            if (@deleted_records) {
268
                $validator = $encoding_validator->('Deleted record');
269
                for my $deleted_record ( grep { $validator->($_) } @deleted_records ) {
270
                    print $deleted_record->as_usmarc();
271
                }
272
            }
273
        } elsif ( $format eq 'xml' ) {
274
            my $marcflavour = C4::Context->preference("marcflavour");
275
            MARC::File::XML->default_record_format(
276
                ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
277
            print MARC::File::XML::header();
278
            print "\n";
279
            for my $record ( @records, @deleted_records ) {
280
                print MARC::File::XML::record($record);
281
                print "\n";
282
            }
283
            print MARC::File::XML::footer();
209
            print "\n";
284
            print "\n";
210
        }
285
        }
211
        print MARC::File::XML::footer();
212
        print "\n";
213
    } elsif ( $format eq 'csv' ) {
286
    } elsif ( $format eq 'csv' ) {
214
        die 'There is no valid csv profile defined for this export'
287
        die 'There is no valid csv profile defined for this export'
215
            unless Koha::CsvProfiles->find($csv_profile_id);
288
            unless Koha::CsvProfiles->find($csv_profile_id);
216
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
289
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
217
    }
290
    }
218
219
    close $fh if $output_filepath;
291
    close $fh if $output_filepath;
220
}
292
}
221
293
(-)a/misc/export_records.pl (-26 / +62 lines)
Lines 38-43 use Koha::Reports; Link Here
38
my (
38
my (
39
    $output_format,
39
    $output_format,
40
    $timestamp,
40
    $timestamp,
41
    $include_deleted,
42
    $deleted_only,
41
    $dont_export_items,
43
    $dont_export_items,
42
    $csv_profile_id,
44
    $csv_profile_id,
43
    $deleted_barcodes,
45
    $deleted_barcodes,
Lines 68-73 my ( Link Here
68
GetOptions(
70
GetOptions(
69
    'format=s'                => \$output_format,
71
    'format=s'                => \$output_format,
70
    'date=s'                  => \$timestamp,
72
    'date=s'                  => \$timestamp,
73
    'include_deleted'         => \$include_deleted,
74
    'deleted_only'            => \$deleted_only,
71
    'dont_export_items'       => \$dont_export_items,
75
    'dont_export_items'       => \$dont_export_items,
72
    'csv_profile_id=s'        => \$csv_profile_id,
76
    'csv_profile_id=s'        => \$csv_profile_id,
73
    'deleted_barcodes'        => \$deleted_barcodes,
77
    'deleted_barcodes'        => \$deleted_barcodes,
Lines 103-108 $record_type ||= 'bibs'; Link Here
103
# Retrocompatibility for the format parameter
107
# Retrocompatibility for the format parameter
104
$output_format = 'iso2709' if $output_format eq 'marc';
108
$output_format = 'iso2709' if $output_format eq 'marc';
105
109
110
if ( $include_deleted || $deleted_only ) {
111
    if ( $record_type ne 'bibs' ) {
112
        pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|);
113
    }
114
    if ( !$timestamp ) {
115
        pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|);
116
    }
117
    if ( $output_format eq 'csv' ) {
118
        pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|);
119
    }
120
}
121
106
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
122
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
107
    pod2usage(q|CSV output is only available for biblio records|);
123
    pod2usage(q|CSV output is only available for biblio records|);
108
}
124
}
Lines 164-169 my $dbh = C4::Context->dbh; Link Here
164
open STDOUT, '>', $filename if $filename;
180
open STDOUT, '>', $filename if $filename;
165
181
166
my @record_ids;
182
my @record_ids;
183
my @deleted_record_ids;
167
184
168
$timestamp =
185
$timestamp =
169
    ($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
186
    ($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
Lines 187-218 if ( $record_type eq 'bibs' ) { Link Here
187
            }
204
            }
188
        }
205
        }
189
    } elsif ($timestamp) {
206
    } elsif ($timestamp) {
190
        if ( !$dont_export_items ) {
207
        unless ($deleted_only) {
191
            push @record_ids, $_->{biblionumber} for @{
208
            if ( !$dont_export_items ) {
192
                $dbh->selectall_arrayref(
209
                push @record_ids, $_->{biblionumber} for @{
193
                    q| (
210
                    $dbh->selectall_arrayref(
194
                    SELECT biblio_metadata.biblionumber
211
                        q| (
195
                    FROM biblio_metadata
212
                        SELECT biblio_metadata.biblionumber
196
                      LEFT JOIN items USING(biblionumber)
213
                        FROM biblio_metadata
197
                    WHERE biblio_metadata.timestamp >= ?
214
                          LEFT JOIN items USING(biblionumber)
198
                      OR items.timestamp >= ?
215
                        WHERE biblio_metadata.timestamp >= ?
199
                ) UNION (
216
                          OR items.timestamp >= ?
200
                    SELECT biblio_metadata.biblionumber
217
                    ) UNION (
201
                    FROM biblio_metadata
218
                        SELECT biblio_metadata.biblionumber
202
                      LEFT JOIN deleteditems USING(biblionumber)
219
                        FROM biblio_metadata
203
                    WHERE biblio_metadata.timestamp >= ?
220
                          LEFT JOIN deleteditems USING(biblionumber)
204
                      OR deleteditems.timestamp >= ?
221
                        WHERE biblio_metadata.timestamp >= ?
205
                ) |, { Slice => {} }, ($timestamp) x 4
222
                          OR deleteditems.timestamp >= ?
206
                );
223
                    ) |, { Slice => {} }, ($timestamp) x 4
207
            };
224
                    );
208
        } else {
225
                };
209
            push @record_ids, $_->{biblionumber} for @{
226
            } else {
227
                push @record_ids, $_->{biblionumber} for @{
228
                    $dbh->selectall_arrayref(
229
                        q| (
230
                        SELECT biblio_metadata.biblionumber
231
                        FROM biblio_metadata
232
                        WHERE biblio_metadata.timestamp >= ?
233
                    ) |, { Slice => {} }, $timestamp
234
                    );
235
                };
236
            }
237
        }
238
        if ( $include_deleted || $deleted_only ) {
239
            push @deleted_record_ids, $_->{biblionumber} for @{
210
                $dbh->selectall_arrayref(
240
                $dbh->selectall_arrayref(
211
                    q| (
241
                    q|
212
                    SELECT biblio_metadata.biblionumber
242
                    SELECT `biblionumber`
213
                    FROM biblio_metadata
243
                    FROM `deletedbiblio`
214
                    WHERE biblio_metadata.timestamp >= ?
244
                    WHERE `timestamp` >= ?
215
                ) |, { Slice => {} }, $timestamp
245
                |, { Slice => {} }, $timestamp
216
                );
246
                );
217
            };
247
            };
218
        }
248
        }
Lines 332-337 if ($deleted_barcodes) { Link Here
332
            record_type             => $record_type,
362
            record_type             => $record_type,
333
            record_ids              => \@record_ids,
363
            record_ids              => \@record_ids,
334
            record_conditions       => @marc_conditions ? \@marc_conditions : undef,
364
            record_conditions       => @marc_conditions ? \@marc_conditions : undef,
365
            deleted_record_ids      => \@deleted_record_ids,
335
            format                  => $output_format,
366
            format                  => $output_format,
336
            csv_profile_id          => $csv_profile_id,
367
            csv_profile_id          => $csv_profile_id,
337
            export_items            => ( not $dont_export_items ),
368
            export_items            => ( not $dont_export_items ),
Lines 348-354 export records - This script exports record (biblios or authorities) Link Here
348
379
349
=head1 SYNOPSIS
380
=head1 SYNOPSIS
350
381
351
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
382
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
352
383
353
=head1 OPTIONS
384
=head1 OPTIONS
354
385
Lines 369-374 Print a brief help message. Link Here
369
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
400
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
370
                        have been modified since DATETIME.
401
                        have been modified since DATETIME.
371
402
403
=item B<--include_deleted>
404
405
 --include_deleted      If enabled, when using --date option, deleted records will be included in export as marc records
406
                        with leader record status set to "d" (deleted).
407
372
=item B<--record-type>
408
=item B<--record-type>
373
409
374
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
410
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
(-)a/t/db_dependent/Exporter/Record.t (-13 / +33 lines)
Lines 76-81 if ( $marcflavour eq 'UNIMARC' ) { Link Here
76
    $homebranch_subfield_code = 'a';
76
    $homebranch_subfield_code = 'a';
77
}
77
}
78
78
79
my $deleted_biblio = MARC::Record->new();
80
$deleted_biblio->leader('00136nam a22000617a 4500');
81
$deleted_biblio->append_fields(
82
    MARC::Field->new( '100', ' ', ' ', a => 'Chopra, Deepak' ),
83
    MARC::Field->new( '245', ' ', ' ', a => 'The seven spiritual laws of success' ),
84
);
85
my ($deleted_biblionumber) = AddBiblio( $deleted_biblio, '' );
86
DelBiblio($deleted_biblionumber);
87
79
my $bad_biblio = Koha::Biblio->new()->store();
88
my $bad_biblio = Koha::Biblio->new()->store();
80
Koha::Biblio::Metadata->new(
89
Koha::Biblio::Metadata->new(
81
    { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => $marcflavour } )
90
    { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => $marcflavour } )
Lines 160-174 EOF Link Here
160
};
169
};
161
170
162
subtest 'export xml' => sub {
171
subtest 'export xml' => sub {
163
    plan tests => 3;
172
    plan tests => 4;
164
    my $generated_xml_file = '/tmp/test_export.xml';
173
    my $generated_xml_file = '/tmp/test_export.xml';
165
    warning_like {
174
    warning_like {
166
        Koha::Exporter::Record::export(
175
        Koha::Exporter::Record::export(
167
            {
176
            {
168
                record_type     => 'bibs',
177
                record_type        => 'bibs',
169
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
178
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
170
                format          => 'xml',
179
                deleted_record_ids => [$deleted_biblionumber],
171
                output_filepath => $generated_xml_file,
180
                format             => 'xml',
181
                output_filepath    => $generated_xml_file,
172
            }
182
            }
173
        );
183
        );
174
    }
184
    }
Lines 186-210 subtest 'export xml' => sub { Link Here
186
    while ( my $record = $records->next ) {
196
    while ( my $record = $records->next ) {
187
        push @records, $record;
197
        push @records, $record;
188
    }
198
    }
189
    is( scalar(@records), 2, 'Export XML: 2 records should have been exported' );
199
    is( scalar(@records), 3, 'Export XML: 3 records should have been exported' );
190
    my $second_record = $records[1];
200
    my $second_record = $records[1];
191
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
201
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
192
    $title = Encode::encode( 'UTF-8', $title );
202
    $title = Encode::encode( 'UTF-8', $title );
193
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
203
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
204
205
    my $deleted_record = $records[2];
206
207
    # Leader has the expected value (and record status "d")
208
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
194
};
209
};
195
210
196
subtest 'export iso2709' => sub {
211
subtest 'export iso2709' => sub {
197
    plan tests => 3;
212
    plan tests => 4;
198
    my $generated_mrc_file = '/tmp/test_export.mrc';
213
    my $generated_mrc_file = '/tmp/test_export.mrc';
199
214
200
    # Get all item infos
215
    # Get all item infos
201
    warning_like {
216
    warning_like {
202
        Koha::Exporter::Record::export(
217
        Koha::Exporter::Record::export(
203
            {
218
            {
204
                record_type     => 'bibs',
219
                record_type        => 'bibs',
205
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
220
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
206
                format          => 'iso2709',
221
                deleted_record_ids => [$deleted_biblionumber],
207
                output_filepath => $generated_mrc_file,
222
                format             => 'iso2709',
223
                output_filepath    => $generated_mrc_file,
208
            }
224
            }
209
        );
225
        );
210
    }
226
    }
Lines 215-225 subtest 'export iso2709' => sub { Link Here
215
    while ( my $record = $records->next ) {
231
    while ( my $record = $records->next ) {
216
        push @records, $record;
232
        push @records, $record;
217
    }
233
    }
218
    is( scalar(@records), 2, 'Export ISO2709: 2 records should have been exported' );
234
    is( scalar(@records), 3, 'Export ISO2709: 3 records should have been exported' );
219
    my $second_record = $records[1];
235
    my $second_record = $records[1];
220
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
236
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
221
    $title = Encode::encode( 'UTF-8', $title );
237
    $title = Encode::encode( 'UTF-8', $title );
222
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
238
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
239
240
    my $deleted_record = $records[2];
241
242
    # Leader has the expected value (and record status "d")
243
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
223
};
244
};
224
245
225
subtest 'export without record_type' => sub {
246
subtest 'export without record_type' => sub {
226
- 

Return to bug 20551