View | Details | Raw Unified | Return to bug 20551
Collapse All | Expand All

(-)a/Koha/Exporter/Record.pm (-28 / +104 lines)
Lines 6-16 use MARC::File::USMARC; Link Here
6
6
7
use C4::AuthoritiesMarc;
7
use C4::AuthoritiesMarc;
8
use C4::Biblio qw( GetMarcFromKohaField );
8
use C4::Biblio qw( GetMarcFromKohaField );
9
use C4::Charset;
9
use C4::Record;
10
use C4::Record;
10
use Koha::CsvProfiles;
11
use Koha::CsvProfiles;
12
use Koha::Database;
11
use Koha::Logger;
13
use Koha::Logger;
12
use List::Util qw( all any );
14
use List::Util qw( all any );
13
15
16
use MARC::Record;
17
use MARC::File::XML;
18
14
sub _get_record_for_export {
19
sub _get_record_for_export {
15
    my ($params)           = @_;
20
    my ($params)           = @_;
16
    my $record_type        = $params->{record_type};
21
    my $record_type        = $params->{record_type};
Lines 105-110 sub _get_record_for_export { Link Here
105
    return $record;
110
    return $record;
106
}
111
}
107
112
113
sub _get_deleted_biblio_for_export {
114
    my ($params)           = @_;
115
    my $biblionumber = $params->{biblionumber};
116
    # Creating schema is expensive, allow caller to
117
    # pass it so don't have to recreate for each call
118
    my $resultset = $params->{resultset} || Koha::Database
119
        ->new()
120
        ->schema()
121
        ->resultset('DeletedbiblioMetadata');
122
    my $marc_flavour = C4::Context->preference('marcflavour');
123
    my $biblio_metadata = $resultset->find({
124
        'biblionumber' => $biblionumber,
125
        'format' => 'marcxml',
126
        'marcflavour' => $marc_flavour
127
    });
128
    my $marc_xml = $biblio_metadata->metadata;
129
    $marc_xml = StripNonXmlChars($marc_xml);
130
131
    my $record = eval {
132
        MARC::Record::new_from_xml($marc_xml, 'UTF-8', $marc_flavour)
133
    };
134
    if (!$record) {
135
        Koha::Logger->get->warn(
136
            "Failed to load MARCXML for deleted biblio with biblionumber \"$biblionumber\": $@"
137
        );
138
        return;
139
    }
140
    # Set deleted flag (record status, position 05)
141
    my $leader = $record->leader;
142
    substr $leader, 5, 1, 'd';
143
    $record->leader($leader);
144
    return $record;
145
}
146
108
sub _get_authority_for_export {
147
sub _get_authority_for_export {
109
    my ($params) = @_;
148
    my ($params) = @_;
110
    my $authid = $params->{authid} || return;
149
    my $authid = $params->{authid} || return;
Lines 122-128 sub _get_biblio_for_export { Link Here
122
161
123
    my $record = eval { C4::Biblio::GetMarcBiblio({ biblionumber => $biblionumber }); };
162
    my $record = eval { C4::Biblio::GetMarcBiblio({ biblionumber => $biblionumber }); };
124
163
125
    return if $@ or not defined $record;
164
    if (!$record) {
165
        Koha::Logger->get->warn(
166
            "Failed to load MARCXML for biblio with biblionumber \"$biblionumber\": $@"
167
        );
168
        return;
169
    }
126
170
127
    if ($export_items) {
171
    if ($export_items) {
128
        C4::Biblio::EmbedItemsInMarcBiblio({
172
        C4::Biblio::EmbedItemsInMarcBiblio({
Lines 149-154 sub export { Link Here
149
193
150
    my $record_type        = $params->{record_type};
194
    my $record_type        = $params->{record_type};
151
    my $record_ids         = $params->{record_ids} || [];
195
    my $record_ids         = $params->{record_ids} || [];
196
    my $deleted_record_ids = $params->{deleted_record_ids} || [];
152
    my $format             = $params->{format};
197
    my $format             = $params->{format};
153
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
198
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
154
    my $export_items       = $params->{export_items};
199
    my $export_items       = $params->{export_items};
Lines 160-166 sub export { Link Here
160
        Koha::Logger->get->warn( "No record_type given." );
205
        Koha::Logger->get->warn( "No record_type given." );
161
        return;
206
        return;
162
    }
207
    }
163
    return unless @$record_ids;
208
    return unless (@{$record_ids} || @{$deleted_record_ids} && $format ne 'csv');
164
209
165
    my $fh;
210
    my $fh;
166
    if ( $output_filepath ) {
211
    if ( $output_filepath ) {
Lines 171-210 sub export { Link Here
171
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
216
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
172
    }
217
    }
173
218
174
    if ( $format eq 'iso2709' ) {
219
    if ($format eq 'xml' || $format eq 'iso2709') {
175
        for my $record_id (@$record_ids) {
220
        my @records;
176
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
221
        @records = map {
177
            next unless $record;
222
            my $record = _get_record_for_export({ %{$params}, record_id => $_ });
178
            my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
223
            $record ? $record : ();
179
            if ( $errorcount_on_decode or $@ ) {
224
        } @{$record_ids};
180
                my $msg = "Record $record_id could not be exported. " .
225
181
                    ( $@ // '' );
226
        my @deleted_records;
182
                chomp $msg;
227
        if (@{$deleted_record_ids}) {
183
                Koha::Logger->get->info( $msg );
228
            my $resultset = Koha::Database
184
                next;
229
            ->new()
185
            }
230
            ->schema()
186
            print $record->as_usmarc();
231
            ->resultset('DeletedbiblioMetadata');
232
            @deleted_records = map {
233
                my $record = _get_deleted_biblio_for_export({
234
                    biblionumber => $_,
235
                    resultset => $resultset,
236
                });
237
                $record ? $record : ();
238
            } @{$deleted_record_ids};
187
        }
239
        }
188
    } elsif ( $format eq 'xml' ) {
240
        if ( $format eq 'iso2709' ) {
189
        my $marcflavour = C4::Context->preference("marcflavour");
241
            my $encoding_validator = sub {
190
        MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
242
                my ($record_type) = @_;
191
243
                return sub {
192
        print MARC::File::XML::header();
244
                    my ($record) = @_;
193
        print "\n";
245
                    my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode($record->as_usmarc)->warnings()) };
194
        for my $record_id (@$record_ids) {
246
                    if ($errorcount_on_decode || $@) {
195
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
247
                        my ($id_tag, $id_subfield) = GetMarcFromKohaField('biblio.biblionumber', '');
196
            next unless $record;
248
                        my $record_id = $record->subfield($id_tag, $id_subfield);
197
            print MARC::File::XML::record($record);
249
                        my $msg = "$record_type $record_id could not be USMARC decoded/encoded. " . ($@ // '');
250
                        chomp $msg;
251
                        Koha::Logger->get->warn($msg);
252
                        return 0;
253
                    }
254
                    return 1;
255
                }
256
            };
257
            my $validator = $encoding_validator->('Record');
258
            for my $record (grep { $validator->($_) } @records) {
259
                print $record->as_usmarc();
260
            }
261
            if (@deleted_records) {
262
                $validator = $encoding_validator->('Deleted record');
263
                for my $deleted_record (grep { $validator->($_) } @deleted_records) {
264
                    print $deleted_record->as_usmarc();
265
                }
266
            }
267
        } elsif ( $format eq 'xml' ) {
268
            my $marcflavour = C4::Context->preference("marcflavour");
269
            MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
270
            print MARC::File::XML::header();
271
            print "\n";
272
            for my $record (@records, @deleted_records) {
273
                print MARC::File::XML::record($record);
274
                print "\n";
275
            }
276
            print MARC::File::XML::footer();
198
            print "\n";
277
            print "\n";
199
        }
278
        }
200
        print MARC::File::XML::footer();
201
        print "\n";
202
    } elsif ( $format eq 'csv' ) {
279
    } elsif ( $format eq 'csv' ) {
203
        die 'There is no valid csv profile defined for this export'
280
        die 'There is no valid csv profile defined for this export'
204
            unless Koha::CsvProfiles->find( $csv_profile_id );
281
            unless Koha::CsvProfiles->find( $csv_profile_id );
205
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
282
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
206
    }
283
    }
207
208
    close $fh if $output_filepath;
284
    close $fh if $output_filepath;
209
}
285
}
210
286
(-)a/misc/export_records.pl (-24 / +63 lines)
Lines 36-41 use Koha::DateUtils qw( dt_from_string output_pref ); Link Here
36
my (
36
my (
37
    $output_format,
37
    $output_format,
38
    $timestamp,
38
    $timestamp,
39
    $include_deleted,
40
    $deleted_only,
39
    $dont_export_items,
41
    $dont_export_items,
40
    $csv_profile_id,
42
    $csv_profile_id,
41
    $deleted_barcodes,
43
    $deleted_barcodes,
Lines 60-65 my ( Link Here
60
GetOptions(
62
GetOptions(
61
    'format=s'                => \$output_format,
63
    'format=s'                => \$output_format,
62
    'date=s'                  => \$timestamp,
64
    'date=s'                  => \$timestamp,
65
    'include_deleted'         => \$include_deleted,
66
    'deleted_only'            => \$deleted_only,
63
    'dont_export_items'       => \$dont_export_items,
67
    'dont_export_items'       => \$dont_export_items,
64
    'csv_profile_id=s'        => \$csv_profile_id,
68
    'csv_profile_id=s'        => \$csv_profile_id,
65
    'deleted_barcodes'        => \$deleted_barcodes,
69
    'deleted_barcodes'        => \$deleted_barcodes,
Lines 92-97 $record_type ||= 'bibs'; Link Here
92
# Retrocompatibility for the format parameter
96
# Retrocompatibility for the format parameter
93
$output_format = 'iso2709' if $output_format eq 'marc';
97
$output_format = 'iso2709' if $output_format eq 'marc';
94
98
99
if ($include_deleted || $deleted_only) {
100
   if ($record_type ne 'bibs') {
101
        pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|);
102
    }
103
    if (!$timestamp) {
104
        pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|);
105
    }
106
    if ($output_format eq 'csv') {
107
        pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|);
108
    }
109
}
110
95
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
111
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
96
    pod2usage(q|CSV output is only available for biblio records|);
112
    pod2usage(q|CSV output is only available for biblio records|);
97
}
113
}
Lines 138-171 open STDOUT, '>', $filename if $filename; Link Here
138
154
139
155
140
my @record_ids;
156
my @record_ids;
157
my @deleted_record_ids;
141
158
142
$timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): '';
159
$timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): '';
143
160
144
if ( $record_type eq 'bibs' ) {
161
if ( $record_type eq 'bibs' ) {
145
    if ( $timestamp ) {
162
    if ( $timestamp ) {
146
        if (!$dont_export_items) {
163
        unless ($deleted_only) {
147
            push @record_ids, $_->{biblionumber} for @{
164
            if (!$dont_export_items) {
148
                $dbh->selectall_arrayref(q| (
165
                push @record_ids, $_->{biblionumber} for @{
149
                    SELECT biblio_metadata.biblionumber
166
                    $dbh->selectall_arrayref(q| (
150
                    FROM biblio_metadata
167
                        SELECT biblio_metadata.biblionumber
151
                      LEFT JOIN items USING(biblionumber)
168
                        FROM biblio_metadata
152
                    WHERE biblio_metadata.timestamp >= ?
169
                          LEFT JOIN items USING(biblionumber)
153
                      OR items.timestamp >= ?
170
                        WHERE biblio_metadata.timestamp >= ?
154
                ) UNION (
171
                          OR items.timestamp >= ?
155
                    SELECT biblio_metadata.biblionumber
172
                    ) UNION (
156
                    FROM biblio_metadata
173
                        SELECT biblio_metadata.biblionumber
157
                      LEFT JOIN deleteditems USING(biblionumber)
174
                        FROM biblio_metadata
158
                    WHERE biblio_metadata.timestamp >= ?
175
                          LEFT JOIN deleteditems USING(biblionumber)
159
                      OR deleteditems.timestamp >= ?
176
                        WHERE biblio_metadata.timestamp >= ?
160
                ) |, { Slice => {} }, ( $timestamp ) x 4 );
177
                          OR deleteditems.timestamp >= ?
161
            };
178
                    ) |, { Slice => {} }, ( $timestamp ) x 4 );
162
        } else {
179
                };
163
            push @record_ids, $_->{biblionumber} for @{
180
            } else {
164
                $dbh->selectall_arrayref(q| (
181
                push @record_ids, $_->{biblionumber} for @{
165
                    SELECT biblio_metadata.biblionumber
182
                    $dbh->selectall_arrayref(q| (
166
                    FROM biblio_metadata
183
                        SELECT biblio_metadata.biblionumber
167
                    WHERE biblio_metadata.timestamp >= ?
184
                        FROM biblio_metadata
168
                ) |, { Slice => {} }, $timestamp );
185
                        WHERE biblio_metadata.timestamp >= ?
186
                    ) |, { Slice => {} }, $timestamp );
187
                };
188
            }
189
        }
190
        if ($include_deleted || $deleted_only) {
191
            push @deleted_record_ids, $_->{biblionumber} for @{
192
                $dbh->selectall_arrayref(q|
193
                    SELECT `biblionumber`
194
                    FROM `deletedbiblio`
195
                    WHERE `timestamp` >= ?
196
                |, { Slice => {} }, $timestamp);
169
            };
197
            };
170
        }
198
        }
171
    } else {
199
    } else {
Lines 252-257 else { Link Here
252
        {   record_type        => $record_type,
280
        {   record_type        => $record_type,
253
            record_ids         => \@record_ids,
281
            record_ids         => \@record_ids,
254
            record_conditions  => @marc_conditions ? \@marc_conditions : undef,
282
            record_conditions  => @marc_conditions ? \@marc_conditions : undef,
283
            deleted_record_ids => \@deleted_record_ids,
255
            format             => $output_format,
284
            format             => $output_format,
256
            csv_profile_id     => $csv_profile_id,
285
            csv_profile_id     => $csv_profile_id,
257
            export_items       => (not $dont_export_items),
286
            export_items       => (not $dont_export_items),
Lines 268-274 export records - This script exports record (biblios or authorities) Link Here
268
297
269
=head1 SYNOPSIS
298
=head1 SYNOPSIS
270
299
271
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
300
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
272
301
273
=head1 OPTIONS
302
=head1 OPTIONS
274
303
Lines 289-294 Print a brief help message. Link Here
289
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
318
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
290
                        have been modified since DATETIME.
319
                        have been modified since DATETIME.
291
320
321
=item B<--include_deleted>
322
323
 --include_deleted      If enabled, when using --date option, deleted records will be included in export as marc records
324
                        with leader record status set to "d" (deleted).
325
326
=item B<--include_deleted>
327
328
 --include_deleted      If enabled, when using --date option, only deleted records will be included in export as marc
329
                        records with leader record status set to "d" (deleted).
330
292
=item B<--record-type>
331
=item B<--record-type>
293
332
294
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
333
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
(-)a/t/db_dependent/Exporter/Record.t (-13 / +31 lines)
Lines 58-63 $biblio_2->append_fields( Link Here
58
);
58
);
59
my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, '');
59
my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, '');
60
60
61
my $deleted_biblio = MARC::Record->new();
62
$deleted_biblio->leader('00136nam a22000617a 4500');
63
$deleted_biblio->append_fields(
64
    MARC::Field->new('100', ' ', ' ', a => 'Chopra, Deepak'),
65
    MARC::Field->new('245', ' ', ' ', a => 'The seven spiritual laws of success'),
66
);
67
my ($deleted_biblionumber) = AddBiblio($deleted_biblio, '');
68
DelBiblio($deleted_biblionumber);
69
61
my $bad_biblio = Koha::Biblio->new()->store();
70
my $bad_biblio = Koha::Biblio->new()->store();
62
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => C4::Context->preference('marcflavour') } )->store();
71
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', schema => C4::Context->preference('marcflavour') } )->store();
63
my $bad_biblionumber = $bad_biblio->id;
72
my $bad_biblionumber = $bad_biblio->id;
Lines 134-147 EOF Link Here
134
};
143
};
135
144
136
subtest 'export xml' => sub {
145
subtest 'export xml' => sub {
137
    plan tests => 3;
146
    plan tests => 4;
138
    my $generated_xml_file = '/tmp/test_export.xml';
147
    my $generated_xml_file = '/tmp/test_export.xml';
139
    warning_like {
148
    warning_like {
140
        Koha::Exporter::Record::export(
149
        Koha::Exporter::Record::export(
141
            {   record_type     => 'bibs',
150
            {   record_type        => 'bibs',
142
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
151
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
143
                format          => 'xml',
152
                deleted_record_ids => [ $deleted_biblionumber ],
144
                output_filepath => $generated_xml_file,
153
                format             => 'xml',
154
                output_filepath    => $generated_xml_file,
145
            }
155
            }
146
        );
156
        );
147
    }
157
    }
Lines 158-180 subtest 'export xml' => sub { Link Here
158
    while ( my $record = $records->next ) {
168
    while ( my $record = $records->next ) {
159
        push @records, $record;
169
        push @records, $record;
160
    }
170
    }
161
    is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' );
171
    is( scalar( @records ), 3, 'Export XML: 3 records should have been exported' );
162
    my $second_record = $records[1];
172
    my $second_record = $records[1];
163
    my $title = $second_record->subfield(245, 'a');
173
    my $title = $second_record->subfield(245, 'a');
164
    $title = Encode::encode('UTF-8', $title);
174
    $title = Encode::encode('UTF-8', $title);
165
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
175
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
176
177
    my $deleted_record = $records[2];
178
    # Leader has the expected value (and record status "d")
179
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
166
};
180
};
167
181
168
subtest 'export iso2709' => sub {
182
subtest 'export iso2709' => sub {
169
    plan tests => 3;
183
    plan tests => 4;
170
    my $generated_mrc_file = '/tmp/test_export.mrc';
184
    my $generated_mrc_file = '/tmp/test_export.mrc';
171
    # Get all item infos
185
    # Get all item infos
172
    warning_like {
186
    warning_like {
173
        Koha::Exporter::Record::export(
187
        Koha::Exporter::Record::export(
174
            {   record_type     => 'bibs',
188
            {   record_type        => 'bibs',
175
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
189
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
176
                format          => 'iso2709',
190
                deleted_record_ids => [ $deleted_biblionumber ],
177
                output_filepath => $generated_mrc_file,
191
                format             => 'iso2709',
192
                output_filepath    => $generated_mrc_file,
178
            }
193
            }
179
        );
194
        );
180
    }
195
    }
Lines 185-195 subtest 'export iso2709' => sub { Link Here
185
    while ( my $record = $records->next ) {
200
    while ( my $record = $records->next ) {
186
        push @records, $record;
201
        push @records, $record;
187
    }
202
    }
188
    is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' );
203
    is( scalar( @records ), 3, 'Export ISO2709: 3 records should have been exported' );
189
    my $second_record = $records[1];
204
    my $second_record = $records[1];
190
    my $title = $second_record->subfield(245, 'a');
205
    my $title = $second_record->subfield(245, 'a');
191
    $title = Encode::encode('UTF-8', $title);
206
    $title = Encode::encode('UTF-8', $title);
192
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
207
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
208
209
    my $deleted_record = $records[2];
210
    # Leader has the expected value (and record status "d")
211
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
193
};
212
};
194
213
195
subtest 'export without record_type' => sub {
214
subtest 'export without record_type' => sub {
196
- 

Return to bug 20551