View | Details | Raw Unified | Return to bug 20551
Collapse All | Expand All

(-)a/Koha/Exporter/Record.pm (-28 / +104 lines)
Lines 6-18 use MARC::File::USMARC; Link Here
6
6
7
use C4::AuthoritiesMarc;
7
use C4::AuthoritiesMarc;
8
use C4::Biblio qw( GetMarcFromKohaField );
8
use C4::Biblio qw( GetMarcFromKohaField );
9
use C4::Charset;
9
use C4::Record;
10
use C4::Record;
10
use Koha::Biblios;
11
use Koha::Biblios;
11
use Koha::CsvProfiles;
12
use Koha::CsvProfiles;
13
use Koha::Database;
12
use Koha::Logger;
14
use Koha::Logger;
13
use Koha::RecordProcessor;
15
use Koha::RecordProcessor;
14
use List::Util qw( all any );
16
use List::Util qw( all any );
15
17
18
use MARC::Record;
19
use MARC::File::XML;
20
16
sub _get_record_for_export {
21
sub _get_record_for_export {
17
    my ($params)           = @_;
22
    my ($params)           = @_;
18
    my $record_type        = $params->{record_type};
23
    my $record_type        = $params->{record_type};
Lines 107-112 sub _get_record_for_export { Link Here
107
    return $record;
112
    return $record;
108
}
113
}
109
114
115
sub _get_deleted_biblio_for_export {
116
    my ($params)           = @_;
117
    my $biblionumber = $params->{biblionumber};
118
    # Creating schema is expensive, allow caller to
119
    # pass it so don't have to recreate for each call
120
    my $resultset = $params->{resultset} || Koha::Database
121
        ->new()
122
        ->schema()
123
        ->resultset('DeletedbiblioMetadata');
124
    my $marc_flavour = C4::Context->preference('marcflavour');
125
    my $biblio_metadata = $resultset->find({
126
        'biblionumber' => $biblionumber,
127
        'format' => 'marcxml',
128
        'marcflavour' => $marc_flavour
129
    });
130
    my $marc_xml = $biblio_metadata->metadata;
131
    $marc_xml = StripNonXmlChars($marc_xml);
132
133
    my $record = eval {
134
        MARC::Record::new_from_xml($marc_xml, 'UTF-8', $marc_flavour)
135
    };
136
    if (!$record) {
137
        Koha::Logger->get->warn(
138
            "Failed to load MARCXML for deleted biblio with biblionumber \"$biblionumber\": $@"
139
        );
140
        return;
141
    }
142
    # Set deleted flag (record status, position 05)
143
    my $leader = $record->leader;
144
    substr $leader, 5, 1, 'd';
145
    $record->leader($leader);
146
    return $record;
147
}
148
110
sub _get_authority_for_export {
149
sub _get_authority_for_export {
111
    my ($params) = @_;
150
    my ($params) = @_;
112
    my $authid = $params->{authid} || return;
151
    my $authid = $params->{authid} || return;
Lines 126-132 sub _get_biblio_for_export { Link Here
126
    my $biblio = Koha::Biblios->find($biblionumber);
165
    my $biblio = Koha::Biblios->find($biblionumber);
127
    my $record = eval { $biblio->metadata->record };
166
    my $record = eval { $biblio->metadata->record };
128
167
129
    return if $@ or not defined $record;
168
    if (!$record) {
169
        Koha::Logger->get->warn(
170
            "Failed to load MARCXML for biblio with biblionumber \"$biblionumber\": $@"
171
        );
172
        return;
173
    }
130
174
131
    if ($embed_see_from_headings) {
175
    if ($embed_see_from_headings) {
132
        my $record_processor = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
176
        my $record_processor = Koha::RecordProcessor->new( { filters => 'EmbedSeeFromHeadings' } );
Lines 162-167 sub export { Link Here
162
206
163
    my $record_type        = $params->{record_type};
207
    my $record_type        = $params->{record_type};
164
    my $record_ids         = $params->{record_ids} || [];
208
    my $record_ids         = $params->{record_ids} || [];
209
    my $deleted_record_ids = $params->{deleted_record_ids} || [];
165
    my $format             = $params->{format};
210
    my $format             = $params->{format};
166
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
211
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
167
    my $export_items       = $params->{export_items};
212
    my $export_items       = $params->{export_items};
Lines 173-179 sub export { Link Here
173
        Koha::Logger->get->warn( "No record_type given." );
218
        Koha::Logger->get->warn( "No record_type given." );
174
        return;
219
        return;
175
    }
220
    }
176
    return unless @$record_ids;
221
    return unless (@{$record_ids} || @{$deleted_record_ids} && $format ne 'csv');
177
222
178
    my $fh;
223
    my $fh;
179
    if ( $output_filepath ) {
224
    if ( $output_filepath ) {
Lines 184-223 sub export { Link Here
184
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
229
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
185
    }
230
    }
186
231
187
    if ( $format eq 'iso2709' ) {
232
    if ($format eq 'xml' || $format eq 'iso2709') {
188
        for my $record_id (@$record_ids) {
233
        my @records;
189
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
234
        @records = map {
190
            next unless $record;
235
            my $record = _get_record_for_export({ %{$params}, record_id => $_ });
191
            my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
236
            $record ? $record : ();
192
            if ( $errorcount_on_decode or $@ ) {
237
        } @{$record_ids};
193
                my $msg = "Record $record_id could not be exported. " .
238
194
                    ( $@ // '' );
239
        my @deleted_records;
195
                chomp $msg;
240
        if (@{$deleted_record_ids}) {
196
                Koha::Logger->get->info( $msg );
241
            my $resultset = Koha::Database
197
                next;
242
            ->new()
198
            }
243
            ->schema()
199
            print $record->as_usmarc();
244
            ->resultset('DeletedbiblioMetadata');
245
            @deleted_records = map {
246
                my $record = _get_deleted_biblio_for_export({
247
                    biblionumber => $_,
248
                    resultset => $resultset,
249
                });
250
                $record ? $record : ();
251
            } @{$deleted_record_ids};
200
        }
252
        }
201
    } elsif ( $format eq 'xml' ) {
253
        if ( $format eq 'iso2709' ) {
202
        my $marcflavour = C4::Context->preference("marcflavour");
254
            my $encoding_validator = sub {
203
        MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
255
                my ($record_type) = @_;
204
256
                return sub {
205
        print MARC::File::XML::header();
257
                    my ($record) = @_;
206
        print "\n";
258
                    my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode($record->as_usmarc)->warnings()) };
207
        for my $record_id (@$record_ids) {
259
                    if ($errorcount_on_decode || $@) {
208
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
260
                        my ($id_tag, $id_subfield) = GetMarcFromKohaField('biblio.biblionumber', '');
209
            next unless $record;
261
                        my $record_id = $record->subfield($id_tag, $id_subfield);
210
            print MARC::File::XML::record($record);
262
                        my $msg = "$record_type $record_id could not be USMARC decoded/encoded. " . ($@ // '');
263
                        chomp $msg;
264
                        Koha::Logger->get->warn($msg);
265
                        return 0;
266
                    }
267
                    return 1;
268
                }
269
            };
270
            my $validator = $encoding_validator->('Record');
271
            for my $record (grep { $validator->($_) } @records) {
272
                print $record->as_usmarc();
273
            }
274
            if (@deleted_records) {
275
                $validator = $encoding_validator->('Deleted record');
276
                for my $deleted_record (grep { $validator->($_) } @deleted_records) {
277
                    print $deleted_record->as_usmarc();
278
                }
279
            }
280
        } elsif ( $format eq 'xml' ) {
281
            my $marcflavour = C4::Context->preference("marcflavour");
282
            MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
283
            print MARC::File::XML::header();
284
            print "\n";
285
            for my $record (@records, @deleted_records) {
286
                print MARC::File::XML::record($record);
287
                print "\n";
288
            }
289
            print MARC::File::XML::footer();
211
            print "\n";
290
            print "\n";
212
        }
291
        }
213
        print MARC::File::XML::footer();
214
        print "\n";
215
    } elsif ( $format eq 'csv' ) {
292
    } elsif ( $format eq 'csv' ) {
216
        die 'There is no valid csv profile defined for this export'
293
        die 'There is no valid csv profile defined for this export'
217
            unless Koha::CsvProfiles->find( $csv_profile_id );
294
            unless Koha::CsvProfiles->find( $csv_profile_id );
218
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
295
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
219
    }
296
    }
220
221
    close $fh if $output_filepath;
297
    close $fh if $output_filepath;
222
}
298
}
223
299
(-)a/misc/export_records.pl (-28 / +58 lines)
Lines 38-43 use Koha::Reports; Link Here
38
my (
38
my (
39
    $output_format,
39
    $output_format,
40
    $timestamp,
40
    $timestamp,
41
    $include_deleted,
42
    $deleted_only,
41
    $dont_export_items,
43
    $dont_export_items,
42
    $csv_profile_id,
44
    $csv_profile_id,
43
    $deleted_barcodes,
45
    $deleted_barcodes,
Lines 68-73 my ( Link Here
68
GetOptions(
70
GetOptions(
69
    'format=s'                => \$output_format,
71
    'format=s'                => \$output_format,
70
    'date=s'                  => \$timestamp,
72
    'date=s'                  => \$timestamp,
73
    'include_deleted'         => \$include_deleted,
74
    'deleted_only'            => \$deleted_only,
71
    'dont_export_items'       => \$dont_export_items,
75
    'dont_export_items'       => \$dont_export_items,
72
    'csv_profile_id=s'        => \$csv_profile_id,
76
    'csv_profile_id=s'        => \$csv_profile_id,
73
    'deleted_barcodes'        => \$deleted_barcodes,
77
    'deleted_barcodes'        => \$deleted_barcodes,
Lines 103-108 $record_type ||= 'bibs'; Link Here
103
# Retrocompatibility for the format parameter
107
# Retrocompatibility for the format parameter
104
$output_format = 'iso2709' if $output_format eq 'marc';
108
$output_format = 'iso2709' if $output_format eq 'marc';
105
109
110
if ($include_deleted || $deleted_only) {
111
   if ($record_type ne 'bibs') {
112
        pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|);
113
    }
114
    if (!$timestamp) {
115
        pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|);
116
    }
117
    if ($output_format eq 'csv') {
118
        pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|);
119
    }
120
}
121
106
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
122
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
107
    pod2usage(q|CSV output is only available for biblio records|);
123
    pod2usage(q|CSV output is only available for biblio records|);
108
}
124
}
Lines 164-169 my $dbh = C4::Context->dbh; Link Here
164
open STDOUT, '>', $filename if $filename;
180
open STDOUT, '>', $filename if $filename;
165
181
166
my @record_ids;
182
my @record_ids;
183
my @deleted_record_ids;
167
184
168
$timestamp =
185
$timestamp =
169
    ($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
186
    ($timestamp) ? output_pref( { dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, } ) : '';
Lines 187-219 if ( $record_type eq 'bibs' ) { Link Here
187
            }
204
            }
188
        }
205
        }
189
    } elsif ($timestamp) {
206
    } elsif ($timestamp) {
190
        if ( !$dont_export_items ) {
207
        unless ($deleted_only) {
191
            push @record_ids, $_->{biblionumber} for @{
208
            if (!$dont_export_items) {
192
                $dbh->selectall_arrayref(
209
                push @record_ids, $_->{biblionumber} for @{
193
                    q| (
210
                    $dbh->selectall_arrayref(q| (
194
                    SELECT biblio_metadata.biblionumber
211
                        SELECT biblio_metadata.biblionumber
195
                    FROM biblio_metadata
212
                        FROM biblio_metadata
196
                      LEFT JOIN items USING(biblionumber)
213
                          LEFT JOIN items USING(biblionumber)
197
                    WHERE biblio_metadata.timestamp >= ?
214
                        WHERE biblio_metadata.timestamp >= ?
198
                      OR items.timestamp >= ?
215
                          OR items.timestamp >= ?
199
                ) UNION (
216
                    ) UNION (
200
                    SELECT biblio_metadata.biblionumber
217
                        SELECT biblio_metadata.biblionumber
201
                    FROM biblio_metadata
218
                        FROM biblio_metadata
202
                      LEFT JOIN deleteditems USING(biblionumber)
219
                          LEFT JOIN deleteditems USING(biblionumber)
203
                    WHERE biblio_metadata.timestamp >= ?
220
                        WHERE biblio_metadata.timestamp >= ?
204
                      OR deleteditems.timestamp >= ?
221
                          OR deleteditems.timestamp >= ?
205
                ) |, { Slice => {} }, ($timestamp) x 4
222
                    ) |, { Slice => {} }, ( $timestamp ) x 4 );
206
                );
223
                };
207
            };
224
            } else {
208
        } else {
225
                push @record_ids, $_->{biblionumber} for @{
209
            push @record_ids, $_->{biblionumber} for @{
226
                    $dbh->selectall_arrayref(q| (
210
                $dbh->selectall_arrayref(
227
                        SELECT biblio_metadata.biblionumber
211
                    q| (
228
                        FROM biblio_metadata
212
                    SELECT biblio_metadata.biblionumber
229
                        WHERE biblio_metadata.timestamp >= ?
213
                    FROM biblio_metadata
230
                    ) |, { Slice => {} }, $timestamp );
214
                    WHERE biblio_metadata.timestamp >= ?
231
                };
215
                ) |, { Slice => {} }, $timestamp
232
            }
216
                );
233
        }
234
        if ($include_deleted || $deleted_only) {
235
            push @deleted_record_ids, $_->{biblionumber} for @{
236
                $dbh->selectall_arrayref(q|
237
                    SELECT `biblionumber`
238
                    FROM `deletedbiblio`
239
                    WHERE `timestamp` >= ?
240
                |, { Slice => {} }, $timestamp);
217
            };
241
            };
218
        }
242
        }
219
    } else {
243
    } else {
Lines 332-337 if ($deleted_barcodes) { Link Here
332
            record_type             => $record_type,
356
            record_type             => $record_type,
333
            record_ids              => \@record_ids,
357
            record_ids              => \@record_ids,
334
            record_conditions       => @marc_conditions ? \@marc_conditions : undef,
358
            record_conditions       => @marc_conditions ? \@marc_conditions : undef,
359
            deleted_record_ids      => \@deleted_record_ids,
335
            format                  => $output_format,
360
            format                  => $output_format,
336
            csv_profile_id          => $csv_profile_id,
361
            csv_profile_id          => $csv_profile_id,
337
            export_items            => ( not $dont_export_items ),
362
            export_items            => ( not $dont_export_items ),
Lines 348-354 export records - This script exports record (biblios or authorities) Link Here
348
373
349
=head1 SYNOPSIS
374
=head1 SYNOPSIS
350
375
351
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
376
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
352
377
353
=head1 OPTIONS
378
=head1 OPTIONS
354
379
Lines 369-374 Print a brief help message. Link Here
369
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
394
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
370
                        have been modified since DATETIME.
395
                        have been modified since DATETIME.
371
396
397
=item B<--include_deleted>
398
399
 --include_deleted      If enabled, when using --date option, deleted records will be included in export as marc records
400
                        with leader record status set to "d" (deleted).
401
372
=item B<--record-type>
402
=item B<--record-type>
373
403
374
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
404
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
(-)a/t/db_dependent/Exporter/Record.t (-13 / +31 lines)
Lines 76-81 if ($marcflavour eq 'UNIMARC') { Link Here
76
    $homebranch_subfield_code = 'a';
76
    $homebranch_subfield_code = 'a';
77
}
77
}
78
78
79
my $deleted_biblio = MARC::Record->new();
80
$deleted_biblio->leader('00136nam a22000617a 4500');
81
$deleted_biblio->append_fields(
82
    MARC::Field->new('100', ' ', ' ', a => 'Chopra, Deepak'),
83
    MARC::Field->new('245', ' ', ' ', a => 'The seven spiritual laws of success'),
84
);
85
my ($deleted_biblionumber) = AddBiblio($deleted_biblio, '');
86
DelBiblio($deleted_biblionumber);
87
79
my $bad_biblio = Koha::Biblio->new()->store();
88
my $bad_biblio = Koha::Biblio->new()->store();
80
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => $marcflavour } )->store();
89
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => $marcflavour } )->store();
81
my $bad_biblionumber = $bad_biblio->id;
90
my $bad_biblionumber = $bad_biblio->id;
Lines 151-164 EOF Link Here
151
};
160
};
152
161
153
subtest 'export xml' => sub {
162
subtest 'export xml' => sub {
154
    plan tests => 3;
163
    plan tests => 4;
155
    my $generated_xml_file = '/tmp/test_export.xml';
164
    my $generated_xml_file = '/tmp/test_export.xml';
156
    warning_like {
165
    warning_like {
157
        Koha::Exporter::Record::export(
166
        Koha::Exporter::Record::export(
158
            {   record_type     => 'bibs',
167
            {   record_type        => 'bibs',
159
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
168
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
160
                format          => 'xml',
169
                deleted_record_ids => [ $deleted_biblionumber ],
161
                output_filepath => $generated_xml_file,
170
                format             => 'xml',
171
                output_filepath    => $generated_xml_file,
162
            }
172
            }
163
        );
173
        );
164
    }
174
    }
Lines 175-197 subtest 'export xml' => sub { Link Here
175
    while ( my $record = $records->next ) {
185
    while ( my $record = $records->next ) {
176
        push @records, $record;
186
        push @records, $record;
177
    }
187
    }
178
    is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' );
188
    is( scalar( @records ), 3, 'Export XML: 3 records should have been exported' );
179
    my $second_record = $records[1];
189
    my $second_record = $records[1];
180
    my $title = $second_record->subfield($title_field_tag, 'a');
190
    my $title = $second_record->subfield($title_field_tag, 'a');
181
    $title = Encode::encode('UTF-8', $title);
191
    $title = Encode::encode('UTF-8', $title);
182
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
192
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
193
194
    my $deleted_record = $records[2];
195
    # Leader has the expected value (and record status "d")
196
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
183
};
197
};
184
198
185
subtest 'export iso2709' => sub {
199
subtest 'export iso2709' => sub {
186
    plan tests => 3;
200
    plan tests => 4;
187
    my $generated_mrc_file = '/tmp/test_export.mrc';
201
    my $generated_mrc_file = '/tmp/test_export.mrc';
188
    # Get all item infos
202
    # Get all item infos
189
    warning_like {
203
    warning_like {
190
        Koha::Exporter::Record::export(
204
        Koha::Exporter::Record::export(
191
            {   record_type     => 'bibs',
205
            {   record_type        => 'bibs',
192
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
206
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
193
                format          => 'iso2709',
207
                deleted_record_ids => [ $deleted_biblionumber ],
194
                output_filepath => $generated_mrc_file,
208
                format             => 'iso2709',
209
                output_filepath    => $generated_mrc_file,
195
            }
210
            }
196
        );
211
        );
197
    }
212
    }
Lines 202-212 subtest 'export iso2709' => sub { Link Here
202
    while ( my $record = $records->next ) {
217
    while ( my $record = $records->next ) {
203
        push @records, $record;
218
        push @records, $record;
204
    }
219
    }
205
    is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' );
220
    is( scalar( @records ), 3, 'Export ISO2709: 3 records should have been exported' );
206
    my $second_record = $records[1];
221
    my $second_record = $records[1];
207
    my $title = $second_record->subfield($title_field_tag, 'a');
222
    my $title = $second_record->subfield($title_field_tag, 'a');
208
    $title = Encode::encode('UTF-8', $title);
223
    $title = Encode::encode('UTF-8', $title);
209
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
224
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
225
226
    my $deleted_record = $records[2];
227
    # Leader has the expected value (and record status "d")
228
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
210
};
229
};
211
230
212
subtest 'export without record_type' => sub {
231
subtest 'export without record_type' => sub {
213
- 

Return to bug 20551