View | Details | Raw Unified | Return to bug 20551
Collapse All | Expand All

(-)a/Koha/Exporter/Record.pm (-29 / +101 lines)
Lines 6-18 use MARC::File::USMARC; Link Here
6
6
7
use C4::AuthoritiesMarc;
7
use C4::AuthoritiesMarc;
8
use C4::Biblio qw( GetMarcFromKohaField );
8
use C4::Biblio qw( GetMarcFromKohaField );
9
use C4::Charset;
9
use C4::Record;
10
use C4::Record;
10
use Koha::Biblios;
11
use Koha::Biblios;
11
use Koha::CsvProfiles;
12
use Koha::CsvProfiles;
13
use Koha::Database;
12
use Koha::Logger;
14
use Koha::Logger;
13
use Koha::RecordProcessor;
15
use Koha::RecordProcessor;
14
use List::Util qw( all any );
16
use List::Util qw( all any );
15
17
18
use MARC::Record;
19
use MARC::File::XML;
20
16
sub _get_record_for_export {
21
sub _get_record_for_export {
17
    my ($params)           = @_;
22
    my ($params)           = @_;
18
    my $record_type        = $params->{record_type};
23
    my $record_type        = $params->{record_type};
Lines 105-110 sub _get_record_for_export { Link Here
105
    return $record;
110
    return $record;
106
}
111
}
107
112
113
sub _get_deleted_biblio_for_export {
114
    my ($params) = @_;
115
    my $biblionumber = $params->{biblionumber};
116
117
    # Creating schema is expensive, allow caller to
118
    # pass it so don't have to recreate for each call
119
    my $resultset       = $params->{resultset} || Koha::Database->new()->schema()->resultset('DeletedbiblioMetadata');
120
    my $marc_flavour    = C4::Context->preference('marcflavour');
121
    my $biblio_metadata = $resultset->find(
122
        {
123
            'biblionumber' => $biblionumber,
124
            'format'       => 'marcxml',
125
            'marcflavour'  => $marc_flavour
126
        }
127
    );
128
    my $marc_xml = $biblio_metadata->metadata;
129
    $marc_xml = StripNonXmlChars($marc_xml);
130
131
    my $record = eval { MARC::Record::new_from_xml( $marc_xml, 'UTF-8', $marc_flavour ) };
132
    if ( !$record ) {
133
        Koha::Logger->get->warn("Failed to load MARCXML for deleted biblio with biblionumber \"$biblionumber\": $@");
134
        return;
135
    }
136
137
    # Set deleted flag (record status, position 05)
138
    my $leader = $record->leader;
139
    substr $leader, 5, 1, 'd';
140
    $record->leader($leader);
141
    return $record;
142
}
143
108
sub _get_authority_for_export {
144
sub _get_authority_for_export {
109
    my ($params)  = @_;
145
    my ($params)  = @_;
110
    my $authid    = $params->{authid} || return;
146
    my $authid    = $params->{authid} || return;
Lines 124-130 sub _get_biblio_for_export { Link Here
124
    my $biblio = Koha::Biblios->find($biblionumber);
160
    my $biblio = Koha::Biblios->find($biblionumber);
125
    my $record = eval { $biblio->metadata->record };
161
    my $record = eval { $biblio->metadata->record };
126
162
127
    return if $@ or not defined $record;
163
    if ( !$record ) {
164
        Koha::Logger->get->warn("Failed to load MARCXML for biblio with biblionumber \"$biblionumber\": $@");
165
        return;
166
    }
128
167
129
    if ($embed_see_from_headings) {
168
    if ($embed_see_from_headings) {
130
        my $record_processor = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
169
        my $record_processor = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
Lines 159-165 sub export { Link Here
159
    my ($params) = @_;
198
    my ($params) = @_;
160
199
161
    my $record_type        = $params->{record_type};
200
    my $record_type        = $params->{record_type};
162
    my $record_ids         = $params->{record_ids} || [];
201
    my $record_ids         = $params->{record_ids}         || [];
202
    my $deleted_record_ids = $params->{deleted_record_ids} || [];
163
    my $format             = $params->{format};
203
    my $format             = $params->{format};
164
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
204
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
165
    my $export_items       = $params->{export_items};
205
    my $export_items       = $params->{export_items};
Lines 171-177 sub export { Link Here
171
        Koha::Logger->get->warn("No record_type given.");
211
        Koha::Logger->get->warn("No record_type given.");
172
        return;
212
        return;
173
    }
213
    }
174
    return unless @$record_ids;
214
    return unless ( @{$record_ids} || @{$deleted_record_ids} && $format ne 'csv' );
175
215
176
    my $fh;
216
    my $fh;
177
    if ($output_filepath) {
217
    if ($output_filepath) {
Lines 182-221 sub export { Link Here
182
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
222
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
183
    }
223
    }
184
224
185
    if ( $format eq 'iso2709' ) {
225
    if ( $format eq 'xml' || $format eq 'iso2709' ) {
186
        for my $record_id (@$record_ids) {
226
        my @records;
187
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
227
        @records = map {
188
            next unless $record;
228
            my $record = _get_record_for_export( { %{$params}, record_id => $_ } );
189
            my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
229
            $record ? $record : ();
190
            if ( $errorcount_on_decode or $@ ) {
230
        } @{$record_ids};
191
                my $msg = "Record $record_id could not be exported. " . ( $@ // '' );
231
192
                chomp $msg;
232
        my @deleted_records;
193
                Koha::Logger->get->info($msg);
233
        if ( @{$deleted_record_ids} ) {
194
                next;
234
            my $resultset = Koha::Database->new()->schema()->resultset('DeletedbiblioMetadata');
195
            }
235
            @deleted_records = map {
196
            print $record->as_usmarc();
236
                my $record = _get_deleted_biblio_for_export(
237
                    {
238
                        biblionumber => $_,
239
                        resultset    => $resultset,
240
                    }
241
                );
242
                $record ? $record : ();
243
            } @{$deleted_record_ids};
197
        }
244
        }
198
    } elsif ( $format eq 'xml' ) {
245
        if ( $format eq 'iso2709' ) {
199
        my $marcflavour = C4::Context->preference("marcflavour");
246
            my $encoding_validator = sub {
200
        MARC::File::XML->default_record_format(
247
                my ($record_type) = @_;
201
            ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
248
                return sub {
202
249
                    my ($record) = @_;
203
        print MARC::File::XML::header();
250
                    my $errorcount_on_decode =
204
        print "\n";
251
                        eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
205
        for my $record_id (@$record_ids) {
252
                    if ( $errorcount_on_decode || $@ ) {
206
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
253
                        my ( $id_tag, $id_subfield ) = GetMarcFromKohaField( 'biblio.biblionumber', '' );
207
            next unless $record;
254
                        my $record_id = $record->subfield( $id_tag, $id_subfield );
208
            print MARC::File::XML::record($record);
255
                        my $msg       = "$record_type $record_id could not be USMARC decoded/encoded. " . ( $@ // '' );
256
                        chomp $msg;
257
                        Koha::Logger->get->warn($msg);
258
                        return 0;
259
                    }
260
                    return 1;
261
                }
262
            };
263
            my $validator = $encoding_validator->('Record');
264
            for my $record ( grep { $validator->($_) } @records ) {
265
                print $record->as_usmarc();
266
            }
267
            if (@deleted_records) {
268
                $validator = $encoding_validator->('Deleted record');
269
                for my $deleted_record ( grep { $validator->($_) } @deleted_records ) {
270
                    print $deleted_record->as_usmarc();
271
                }
272
            }
273
        } elsif ( $format eq 'xml' ) {
274
            my $marcflavour = C4::Context->preference("marcflavour");
275
            MARC::File::XML->default_record_format(
276
                ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
277
            print MARC::File::XML::header();
278
            print "\n";
279
            for my $record ( @records, @deleted_records ) {
280
                print MARC::File::XML::record($record);
281
                print "\n";
282
            }
283
            print MARC::File::XML::footer();
209
            print "\n";
284
            print "\n";
210
        }
285
        }
211
        print MARC::File::XML::footer();
212
        print "\n";
213
    } elsif ( $format eq 'csv' ) {
286
    } elsif ( $format eq 'csv' ) {
214
        die 'There is no valid csv profile defined for this export'
287
        die 'There is no valid csv profile defined for this export'
215
            unless Koha::CsvProfiles->find($csv_profile_id);
288
            unless Koha::CsvProfiles->find($csv_profile_id);
216
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
289
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
217
    }
290
    }
218
219
    close $fh if $output_filepath;
291
    close $fh if $output_filepath;
220
}
292
}
221
293
(-)a/misc/export_records.pl (-26 / +62 lines)
Lines 40-45 use Koha::File::Transports; Link Here
40
my (
40
my (
41
    $output_format,
41
    $output_format,
42
    $timestamp,
42
    $timestamp,
43
    $include_deleted,
44
    $deleted_only,
43
    $dont_export_items,
45
    $dont_export_items,
44
    $csv_profile_id,
46
    $csv_profile_id,
45
    $deleted_barcodes,
47
    $deleted_barcodes,
Lines 72-77 my ( Link Here
72
GetOptions(
74
GetOptions(
73
    'format=s'                => \$output_format,
75
    'format=s'                => \$output_format,
74
    'date=s'                  => \$timestamp,
76
    'date=s'                  => \$timestamp,
77
    'include_deleted'         => \$include_deleted,
78
    'deleted_only'            => \$deleted_only,
75
    'dont_export_items'       => \$dont_export_items,
79
    'dont_export_items'       => \$dont_export_items,
76
    'csv_profile_id=s'        => \$csv_profile_id,
80
    'csv_profile_id=s'        => \$csv_profile_id,
77
    'deleted_barcodes'        => \$deleted_barcodes,
81
    'deleted_barcodes'        => \$deleted_barcodes,
Lines 109-114 $record_type ||= 'bibs'; Link Here
109
# Retrocompatibility for the format parameter
113
# Retrocompatibility for the format parameter
110
$output_format = 'iso2709' if $output_format eq 'marc';
114
$output_format = 'iso2709' if $output_format eq 'marc';
111
115
116
if ( $include_deleted || $deleted_only ) {
117
    if ( $record_type ne 'bibs' ) {
118
        pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|);
119
    }
120
    if ( !$timestamp ) {
121
        pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|);
122
    }
123
    if ( $output_format eq 'csv' ) {
124
        pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|);
125
    }
126
}
127
112
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
128
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
113
    pod2usage(q|CSV output is only available for biblio records|);
129
    pod2usage(q|CSV output is only available for biblio records|);
114
}
130
}
Lines 183-188 my $dbh = C4::Context->dbh; Link Here
183
open STDOUT, '>', $filename if $filename;
199
open STDOUT, '>', $filename if $filename;
184
200
185
my @record_ids;
201
my @record_ids;
202
my @deleted_record_ids;
186
203
187
$timestamp =
204
$timestamp =
188
    ($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
205
    ($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
Lines 206-237 if ( $record_type eq 'bibs' ) { Link Here
206
            }
223
            }
207
        }
224
        }
208
    } elsif ($timestamp) {
225
    } elsif ($timestamp) {
209
        if ( !$dont_export_items ) {
226
        unless ($deleted_only) {
210
            push @record_ids, $_->{biblionumber} for @{
227
            if ( !$dont_export_items ) {
211
                $dbh->selectall_arrayref(
228
                push @record_ids, $_->{biblionumber} for @{
212
                    q| (
229
                    $dbh->selectall_arrayref(
213
                    SELECT biblio_metadata.biblionumber
230
                        q| (
214
                    FROM biblio_metadata
231
                        SELECT biblio_metadata.biblionumber
215
                      LEFT JOIN items USING(biblionumber)
232
                        FROM biblio_metadata
216
                    WHERE biblio_metadata.timestamp >= ?
233
                          LEFT JOIN items USING(biblionumber)
217
                      OR items.timestamp >= ?
234
                        WHERE biblio_metadata.timestamp >= ?
218
                ) UNION (
235
                          OR items.timestamp >= ?
219
                    SELECT biblio_metadata.biblionumber
236
                    ) UNION (
220
                    FROM biblio_metadata
237
                        SELECT biblio_metadata.biblionumber
221
                      LEFT JOIN deleteditems USING(biblionumber)
238
                        FROM biblio_metadata
222
                    WHERE biblio_metadata.timestamp >= ?
239
                          LEFT JOIN deleteditems USING(biblionumber)
223
                      OR deleteditems.timestamp >= ?
240
                        WHERE biblio_metadata.timestamp >= ?
224
                ) |, { Slice => {} }, ($timestamp) x 4
241
                          OR deleteditems.timestamp >= ?
225
                );
242
                    ) |, { Slice => {} }, ($timestamp) x 4
226
            };
243
                    );
227
        } else {
244
                };
228
            push @record_ids, $_->{biblionumber} for @{
245
            } else {
246
                push @record_ids, $_->{biblionumber} for @{
247
                    $dbh->selectall_arrayref(
248
                        q| (
249
                        SELECT biblio_metadata.biblionumber
250
                        FROM biblio_metadata
251
                        WHERE biblio_metadata.timestamp >= ?
252
                    ) |, { Slice => {} }, $timestamp
253
                    );
254
                };
255
            }
256
        }
257
        if ( $include_deleted || $deleted_only ) {
258
            push @deleted_record_ids, $_->{biblionumber} for @{
229
                $dbh->selectall_arrayref(
259
                $dbh->selectall_arrayref(
230
                    q| (
260
                    q|
231
                    SELECT biblio_metadata.biblionumber
261
                    SELECT `biblionumber`
232
                    FROM biblio_metadata
262
                    FROM `deletedbiblio`
233
                    WHERE biblio_metadata.timestamp >= ?
263
                    WHERE `timestamp` >= ?
234
                ) |, { Slice => {} }, $timestamp
264
                |, { Slice => {} }, $timestamp
235
                );
265
                );
236
            };
266
            };
237
        }
267
        }
Lines 351-356 if ($deleted_barcodes) { Link Here
351
            record_type             => $record_type,
381
            record_type             => $record_type,
352
            record_ids              => \@record_ids,
382
            record_ids              => \@record_ids,
353
            record_conditions       => @marc_conditions ? \@marc_conditions : undef,
383
            record_conditions       => @marc_conditions ? \@marc_conditions : undef,
384
            deleted_record_ids      => \@deleted_record_ids,
354
            format                  => $output_format,
385
            format                  => $output_format,
355
            csv_profile_id          => $csv_profile_id,
386
            csv_profile_id          => $csv_profile_id,
356
            export_items            => ( not $dont_export_items ),
387
            export_items            => ( not $dont_export_items ),
Lines 415-421 export records - This script exports record (biblios or authorities) Link Here
415
446
416
=head1 SYNOPSIS
447
=head1 SYNOPSIS
417
448
418
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
449
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
419
450
420
=head1 OPTIONS
451
=head1 OPTIONS
421
452
Lines 436-441 Print a brief help message. Link Here
436
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
467
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
437
                        have been modified since DATETIME.
468
                        have been modified since DATETIME.
438
469
470
=item B<--include_deleted>
471
472
 --include_deleted      If enabled, when using --date option, deleted records will be included in export as marc records
473
                        with leader record status set to "d" (deleted).
474
439
=item B<--record-type>
475
=item B<--record-type>
440
476
441
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
477
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
(-)a/t/db_dependent/Exporter/Record.t (-13 / +33 lines)
Lines 77-82 if ( $marcflavour eq 'UNIMARC' ) { Link Here
77
    $homebranch_subfield_code = 'a';
77
    $homebranch_subfield_code = 'a';
78
}
78
}
79
79
80
my $deleted_biblio = MARC::Record->new();
81
$deleted_biblio->leader('00136nam a22000617a 4500');
82
$deleted_biblio->append_fields(
83
    MARC::Field->new( '100', ' ', ' ', a => 'Chopra, Deepak' ),
84
    MARC::Field->new( '245', ' ', ' ', a => 'The seven spiritual laws of success' ),
85
);
86
my ($deleted_biblionumber) = AddBiblio( $deleted_biblio, '' );
87
DelBiblio($deleted_biblionumber);
88
80
my $bad_biblio = Koha::Biblio->new()->store();
89
my $bad_biblio = Koha::Biblio->new()->store();
81
Koha::Biblio::Metadata->new(
90
Koha::Biblio::Metadata->new(
82
    { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => $marcflavour } )
91
    { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => $marcflavour } )
Lines 161-175 EOF Link Here
161
};
170
};
162
171
163
subtest 'export xml' => sub {
172
subtest 'export xml' => sub {
164
    plan tests => 3;
173
    plan tests => 4;
165
    my $generated_xml_file = '/tmp/test_export.xml';
174
    my $generated_xml_file = '/tmp/test_export.xml';
166
    warning_like {
175
    warning_like {
167
        Koha::Exporter::Record::export(
176
        Koha::Exporter::Record::export(
168
            {
177
            {
169
                record_type     => 'bibs',
178
                record_type        => 'bibs',
170
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
179
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
171
                format          => 'xml',
180
                deleted_record_ids => [$deleted_biblionumber],
172
                output_filepath => $generated_xml_file,
181
                format             => 'xml',
182
                output_filepath    => $generated_xml_file,
173
            }
183
            }
174
        );
184
        );
175
    }
185
    }
Lines 187-211 subtest 'export xml' => sub { Link Here
187
    while ( my $record = $records->next ) {
197
    while ( my $record = $records->next ) {
188
        push @records, $record;
198
        push @records, $record;
189
    }
199
    }
190
    is( scalar(@records), 2, 'Export XML: 2 records should have been exported' );
200
    is( scalar(@records), 3, 'Export XML: 3 records should have been exported' );
191
    my $second_record = $records[1];
201
    my $second_record = $records[1];
192
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
202
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
193
    $title = Encode::encode( 'UTF-8', $title );
203
    $title = Encode::encode( 'UTF-8', $title );
194
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
204
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
205
206
    my $deleted_record = $records[2];
207
208
    # Leader has the expected value (and record status "d")
209
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
195
};
210
};
196
211
197
subtest 'export iso2709' => sub {
212
subtest 'export iso2709' => sub {
198
    plan tests => 3;
213
    plan tests => 4;
199
    my $generated_mrc_file = '/tmp/test_export.mrc';
214
    my $generated_mrc_file = '/tmp/test_export.mrc';
200
215
201
    # Get all item infos
216
    # Get all item infos
202
    warning_like {
217
    warning_like {
203
        Koha::Exporter::Record::export(
218
        Koha::Exporter::Record::export(
204
            {
219
            {
205
                record_type     => 'bibs',
220
                record_type        => 'bibs',
206
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
221
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
207
                format          => 'iso2709',
222
                deleted_record_ids => [$deleted_biblionumber],
208
                output_filepath => $generated_mrc_file,
223
                format             => 'iso2709',
224
                output_filepath    => $generated_mrc_file,
209
            }
225
            }
210
        );
226
        );
211
    }
227
    }
Lines 216-226 subtest 'export iso2709' => sub { Link Here
216
    while ( my $record = $records->next ) {
232
    while ( my $record = $records->next ) {
217
        push @records, $record;
233
        push @records, $record;
218
    }
234
    }
219
    is( scalar(@records), 2, 'Export ISO2709: 2 records should have been exported' );
235
    is( scalar(@records), 3, 'Export ISO2709: 3 records should have been exported' );
220
    my $second_record = $records[1];
236
    my $second_record = $records[1];
221
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
237
    my $title         = $second_record->subfield( $title_field_tag, 'a' );
222
    $title = Encode::encode( 'UTF-8', $title );
238
    $title = Encode::encode( 'UTF-8', $title );
223
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
239
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
240
241
    my $deleted_record = $records[2];
242
243
    # Leader has the expected value (and record status "d")
244
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
224
};
245
};
225
246
226
subtest 'export without record_type' => sub {
247
subtest 'export without record_type' => sub {
227
- 

Return to bug 20551