View | Details | Raw Unified | Return to bug 20551
Collapse All | Expand All

(-)a/Koha/Exporter/Record.pm (-27 / +96 lines)
Lines 6-16 use MARC::File::USMARC; Link Here
6
6
7
use C4::AuthoritiesMarc;
7
use C4::AuthoritiesMarc;
8
use C4::Biblio;
8
use C4::Biblio;
9
use C4::Charset;
9
use C4::Record;
10
use C4::Record;
10
use Koha::CsvProfiles;
11
use Koha::CsvProfiles;
12
use Koha::Database;
11
use Koha::Logger;
13
use Koha::Logger;
12
use List::Util qw(all any);
14
use List::Util qw(all any);
13
15
16
use MARC::Record;
17
use MARC::File::XML;
18
14
sub _get_record_for_export {
19
sub _get_record_for_export {
15
    my ($params)           = @_;
20
    my ($params)           = @_;
16
    my $record_type        = $params->{record_type};
21
    my $record_type        = $params->{record_type};
Lines 105-110 sub _get_record_for_export { Link Here
105
    return $record;
110
    return $record;
106
}
111
}
107
112
113
sub _get_deleted_biblio_for_export {
114
    my ($params)           = @_;
115
    my $record_id = $params->{record_id};
116
    # Creating schema is expensive, allow caller to
117
    # pass it so don't have to recreate for each call
118
    my $resultset = $params->{resultset} || Koha::Database
119
        ->new()
120
        ->schema()
121
        ->resultset('DeletedbiblioMetadata');
122
    my $marc_flavour = C4::Context->preference('marcflavour');
123
    my $biblio_metadata = $resultset->find({
124
        'biblionumber' => $record_id,
125
        'format' => 'marcxml',
126
        'marcflavour' => $marc_flavour
127
    });
128
    my $marc_xml = $biblio_metadata->metadata;
129
    $marc_xml = StripNonXmlChars($marc_xml);
130
131
    my $record = eval {
132
        MARC::Record::new_from_xml($marc_xml, 'UTF-8', $marc_flavour)
133
    };
134
    if ($@) {
135
        warn "Failed to load MARCXML for biblio with biblionumber \"$record_id\": $@";
136
        return undef;
137
    }
138
    # Set deleted flag (record status, position 05)
139
    my $leader = $record->leader;
140
    substr $leader, 5, 1, 'd';
141
    $record->leader($leader);
142
    return $record;
143
}
144
108
sub _get_authority_for_export {
145
sub _get_authority_for_export {
109
    my ($params) = @_;
146
    my ($params) = @_;
110
    my $authid = $params->{authid} || return;
147
    my $authid = $params->{authid} || return;
Lines 149-154 sub export { Link Here
149
186
150
    my $record_type        = $params->{record_type};
187
    my $record_type        = $params->{record_type};
151
    my $record_ids         = $params->{record_ids} || [];
188
    my $record_ids         = $params->{record_ids} || [];
189
    my $deleted_record_ids = $params->{deleted_record_ids} || [];
152
    my $format             = $params->{format};
190
    my $format             = $params->{format};
153
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
191
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
154
    my $export_items       = $params->{export_items};
192
    my $export_items       = $params->{export_items};
Lines 160-166 sub export { Link Here
160
        Koha::Logger->get->warn( "No record_type given." );
198
        Koha::Logger->get->warn( "No record_type given." );
161
        return;
199
        return;
162
    }
200
    }
163
    return unless @$record_ids;
201
    return unless (@{$record_ids} || @{$deleted_record_ids} && $format ne 'csv');
164
202
165
    my $fh;
203
    my $fh;
166
    if ( $output_filepath ) {
204
    if ( $output_filepath ) {
Lines 171-210 sub export { Link Here
171
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
209
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
172
    }
210
    }
173
211
174
    if ( $format eq 'iso2709' ) {
212
    if ($format eq 'xml' || $format eq 'iso2709') {
175
        for my $record_id (@$record_ids) {
213
        my @records;
176
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
214
        @records = map {
177
            next unless $record;
215
            my $record = _get_record_for_export({ %{$params}, record_id => $_ });
178
            my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
216
            $record ? $record : ();
179
            if ( $errorcount_on_decode or $@ ) {
217
        } @{$record_ids};
180
                my $msg = "Record $record_id could not be exported. " .
218
181
                    ( $@ // '' );
219
        my @deleted_records;
182
                chomp $msg;
220
        if (@{$deleted_record_ids}) {
183
                Koha::Logger->get->info( $msg );
221
            my $resultset = Koha::Database
184
                next;
222
            ->new()
185
            }
223
            ->schema()
186
            print $record->as_usmarc();
224
            ->resultset('DeletedbiblioMetadata');
225
            @deleted_records = map {
226
                my $record = _get_deleted_biblio_for_export({
227
                    record_id => $_,
228
                    resultset => $resultset,
229
                });
230
                $record ? $record : ();
231
            } @{$deleted_record_ids};
187
        }
232
        }
188
    } elsif ( $format eq 'xml' ) {
233
        if ( $format eq 'iso2709' ) {
189
        my $marcflavour = C4::Context->preference("marcflavour");
234
            my $encoding_validator = sub {
190
        MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
235
                my ($record_type) = @_;
191
236
                return sub {
192
        print MARC::File::XML::header();
237
                    my ($record) = @_;
193
        print "\n";
238
                    my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode($record->as_usmarc)->warnings()) };
194
        for my $record_id (@$record_ids) {
239
                    if ($errorcount_on_decode || $@) {
195
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
240
                        my ($id_tag, $id_subfield) = GetMarcFromKohaField('biblio.biblionumber', '');
196
            next unless $record;
241
                        my $record_id = $record->subfield($id_tag, $id_subfield);
197
            print MARC::File::XML::record($record);
242
                        my $msg = "$record_type $record_id could not be USMARC decoded/encoded. " . ($@ // '');
243
                        chomp $msg;
244
                        Koha::Logger->get->warn($msg);
245
                        return 0;
246
                    }
247
                    return 1;
248
                }
249
            };
250
            my $validator = $encoding_validator->('Record');
251
            for my $record (grep { $validator->($_) } @records) {
252
                print $record->as_usmarc();
253
            }
254
            if (@deleted_records) {
255
                $validator = $encoding_validator->('Deleted record');
256
                for my $deleted_record (grep { $validator->($_) } @deleted_records) {
257
                    print $deleted_record->as_usmarc();
258
                }
259
            }
260
        } elsif ( $format eq 'xml' ) {
261
            my $marcflavour = C4::Context->preference("marcflavour");
262
            MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
263
            print MARC::File::XML::header();
264
            print "\n";
265
            for my $record (@records, @deleted_records) {
266
                print MARC::File::XML::record($record);
267
                print "\n";
268
            }
269
            print MARC::File::XML::footer();
198
            print "\n";
270
            print "\n";
199
        }
271
        }
200
        print MARC::File::XML::footer();
201
        print "\n";
202
    } elsif ( $format eq 'csv' ) {
272
    } elsif ( $format eq 'csv' ) {
203
        die 'There is no valid csv profile defined for this export'
273
        die 'There is no valid csv profile defined for this export'
204
            unless Koha::CsvProfiles->find( $csv_profile_id );
274
            unless Koha::CsvProfiles->find( $csv_profile_id );
205
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
275
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
206
    }
276
    }
207
208
    close $fh if $output_filepath;
277
    close $fh if $output_filepath;
209
}
278
}
210
279
(-)a/misc/export_records.pl (-16 / +55 lines)
Lines 35-40 use Koha::DateUtils qw( dt_from_string output_pref ); Link Here
35
my (
35
my (
36
    $output_format,
36
    $output_format,
37
    $timestamp,
37
    $timestamp,
38
    $include_deleted,
39
    $deleted_only,
38
    $dont_export_items,
40
    $dont_export_items,
39
    $csv_profile_id,
41
    $csv_profile_id,
40
    $deleted_barcodes,
42
    $deleted_barcodes,
Lines 59-64 my ( Link Here
59
GetOptions(
61
GetOptions(
60
    'format=s'                => \$output_format,
62
    'format=s'                => \$output_format,
61
    'date=s'                  => \$timestamp,
63
    'date=s'                  => \$timestamp,
64
    'include_deleted'         => \$include_deleted,
65
    'deleted_only'            => \$deleted_only,
62
    'dont_export_items'       => \$dont_export_items,
66
    'dont_export_items'       => \$dont_export_items,
63
    'csv_profile_id=s'        => \$csv_profile_id,
67
    'csv_profile_id=s'        => \$csv_profile_id,
64
    'deleted_barcodes'        => \$deleted_barcodes,
68
    'deleted_barcodes'        => \$deleted_barcodes,
Lines 91-96 $record_type ||= 'bibs'; Link Here
91
# Retrocompatibility for the format parameter
95
# Retrocompatibility for the format parameter
92
$output_format = 'iso2709' if $output_format eq 'marc';
96
$output_format = 'iso2709' if $output_format eq 'marc';
93
97
98
if ($include_deleted || $deleted_only) {
99
   if ($record_type ne 'bibs') {
100
        pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|);
101
    }
102
    if (!$timestamp) {
103
        pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|);
104
    }
105
    if ($output_format eq 'csv') {
106
        pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|);
107
    }
108
}
109
94
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
110
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
95
    pod2usage(q|CSV output is only available for biblio records|);
111
    pod2usage(q|CSV output is only available for biblio records|);
96
}
112
}
Lines 137-162 open STDOUT, '>', $filename if $filename; Link Here
137
153
138
154
139
my @record_ids;
155
my @record_ids;
156
my @deleted_record_ids;
140
157
141
$timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): '';
158
$timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): '';
142
159
143
if ( $record_type eq 'bibs' ) {
160
if ( $record_type eq 'bibs' ) {
144
    if ( $timestamp ) {
161
    if ( $timestamp ) {
145
        push @record_ids, $_->{biblionumber} for @{
162
        unless ($deleted_only) {
146
            $dbh->selectall_arrayref(q| (
163
            push @record_ids, $_->{biblionumber} for @{
147
                SELECT biblio_metadata.biblionumber
164
                $dbh->selectall_arrayref(q| (
148
                FROM biblio_metadata
165
                    SELECT biblio_metadata.biblionumber
149
                  LEFT JOIN items USING(biblionumber)
166
                    FROM biblio_metadata
150
                WHERE biblio_metadata.timestamp >= ?
167
                      LEFT JOIN items USING(biblionumber)
151
                  OR items.timestamp >= ?
168
                    WHERE biblio_metadata.timestamp >= ?
152
            ) UNION (
169
                      OR items.timestamp >= ?
153
                SELECT biblio_metadata.biblionumber
170
                ) UNION (
154
                FROM biblio_metadata
171
                    SELECT biblio_metadata.biblionumber
155
                  LEFT JOIN deleteditems USING(biblionumber)
172
                    FROM biblio_metadata
156
                WHERE biblio_metadata.timestamp >= ?
173
                      LEFT JOIN deleteditems USING(biblionumber)
157
                  OR deleteditems.timestamp >= ?
174
                    WHERE biblio_metadata.timestamp >= ?
158
            ) |, { Slice => {} }, ( $timestamp ) x 4 );
175
                      OR deleteditems.timestamp >= ?
159
        };
176
                ) |, { Slice => {} }, ( $timestamp ) x 4 );
177
            };
178
        }
179
        if ($include_deleted || $deleted_only) {
180
            push @deleted_record_ids, $_->{biblionumber} for @{
181
                $dbh->selectall_arrayref(q|
182
                    SELECT `biblionumber`
183
                    FROM `deletedbiblio`
184
                    WHERE `timestamp` >= ?
185
                |, { Slice => {} }, $timestamp);
186
            };
187
        }
160
    } else {
188
    } else {
161
        my $conditions = {
189
        my $conditions = {
162
            ( $starting_biblionumber or $ending_biblionumber )
190
            ( $starting_biblionumber or $ending_biblionumber )
Lines 240-245 else { Link Here
240
        {   record_type        => $record_type,
268
        {   record_type        => $record_type,
241
            record_ids         => \@record_ids,
269
            record_ids         => \@record_ids,
242
            record_conditions  => @marc_conditions ? \@marc_conditions : undef,
270
            record_conditions  => @marc_conditions ? \@marc_conditions : undef,
271
            deleted_record_ids => \@deleted_record_ids,
243
            format             => $output_format,
272
            format             => $output_format,
244
            csv_profile_id     => $csv_profile_id,
273
            csv_profile_id     => $csv_profile_id,
245
            export_items       => (not $dont_export_items),
274
            export_items       => (not $dont_export_items),
Lines 256-262 export records - This script exports record (biblios or authorities) Link Here
256
285
257
=head1 SYNOPSIS
286
=head1 SYNOPSIS
258
287
259
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
288
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
260
289
261
=head1 OPTIONS
290
=head1 OPTIONS
262
291
Lines 277-282 Print a brief help message. Link Here
277
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
306
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
278
                        have been modified since DATETIME.
307
                        have been modified since DATETIME.
279
308
309
=item B<--include_deleted>
310
311
 --include_deleted      If enabled, when using --date option, deleted records will be included in export as marc records
312
                        with leader record status set to "d" (deleted).
313
314
=item B<--include_deleted>
315
316
 --include_deleted      If enabled, when using --date option, only deleted records will be included in export as marc
317
                        records with leader record status set to "d" (deleted).
318
280
=item B<--record-type>
319
=item B<--record-type>
281
320
282
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
321
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
(-)a/t/db_dependent/Exporter/Record.t (-13 / +31 lines)
Lines 58-63 $biblio_2->append_fields( Link Here
58
);
58
);
59
my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, '');
59
my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, '');
60
60
61
my $deleted_biblio = MARC::Record->new();
62
$deleted_biblio->leader('00136nam a22000617a 4500');
63
$deleted_biblio->append_fields(
64
    MARC::Field->new('100', ' ', ' ', a => 'Chopra, Deepak'),
65
    MARC::Field->new('245', ' ', ' ', a => 'The seven spiritual laws of success'),
66
);
67
my ($deleted_biblionumber) = AddBiblio($deleted_biblio, '');
68
DelBiblio($deleted_biblionumber);
69
61
my $bad_biblio = Koha::Biblio->new()->store();
70
my $bad_biblio = Koha::Biblio->new()->store();
62
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', marcflavour => C4::Context->preference('marcflavour') } )->store();
71
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', marcflavour => C4::Context->preference('marcflavour') } )->store();
63
my $bad_biblionumber = $bad_biblio->id;
72
my $bad_biblionumber = $bad_biblio->id;
Lines 141-154 EOF Link Here
141
};
150
};
142
151
143
subtest 'export xml' => sub {
152
subtest 'export xml' => sub {
144
    plan tests => 3;
153
    plan tests => 4;
145
    my $generated_xml_file = '/tmp/test_export.xml';
154
    my $generated_xml_file = '/tmp/test_export.xml';
146
    warning_like {
155
    warning_like {
147
        Koha::Exporter::Record::export(
156
        Koha::Exporter::Record::export(
148
            {   record_type     => 'bibs',
157
            {   record_type        => 'bibs',
149
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
158
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
150
                format          => 'xml',
159
                deleted_record_ids => [ $deleted_biblionumber ],
151
                output_filepath => $generated_xml_file,
160
                format             => 'xml',
161
                output_filepath    => $generated_xml_file,
152
            }
162
            }
153
        );
163
        );
154
    }
164
    }
Lines 165-187 subtest 'export xml' => sub { Link Here
165
    while ( my $record = $records->next ) {
175
    while ( my $record = $records->next ) {
166
        push @records, $record;
176
        push @records, $record;
167
    }
177
    }
168
    is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' );
178
    is( scalar( @records ), 3, 'Export XML: 3 records should have been exported' );
169
    my $second_record = $records[1];
179
    my $second_record = $records[1];
170
    my $title = $second_record->subfield(245, 'a');
180
    my $title = $second_record->subfield(245, 'a');
171
    $title = Encode::encode('UTF-8', $title);
181
    $title = Encode::encode('UTF-8', $title);
172
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
182
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
183
184
    my $deleted_record = $records[2];
185
    # Leader has the expected value (and record status "d")
186
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
173
};
187
};
174
188
175
subtest 'export iso2709' => sub {
189
subtest 'export iso2709' => sub {
176
    plan tests => 3;
190
    plan tests => 4;
177
    my $generated_mrc_file = '/tmp/test_export.mrc';
191
    my $generated_mrc_file = '/tmp/test_export.mrc';
178
    # Get all item infos
192
    # Get all item infos
179
    warning_like {
193
    warning_like {
180
        Koha::Exporter::Record::export(
194
        Koha::Exporter::Record::export(
181
            {   record_type     => 'bibs',
195
            {   record_type        => 'bibs',
182
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
196
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
183
                format          => 'iso2709',
197
                deleted_record_ids => [ $deleted_biblionumber ],
184
                output_filepath => $generated_mrc_file,
198
                format             => 'iso2709',
199
                output_filepath    => $generated_mrc_file,
185
            }
200
            }
186
        );
201
        );
187
    }
202
    }
Lines 192-202 subtest 'export iso2709' => sub { Link Here
192
    while ( my $record = $records->next ) {
207
    while ( my $record = $records->next ) {
193
        push @records, $record;
208
        push @records, $record;
194
    }
209
    }
195
    is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' );
210
    is( scalar( @records ), 3, 'Export ISO2709: 3 records should have been exported' );
196
    my $second_record = $records[1];
211
    my $second_record = $records[1];
197
    my $title = $second_record->subfield(245, 'a');
212
    my $title = $second_record->subfield(245, 'a');
198
    $title = Encode::encode('UTF-8', $title);
213
    $title = Encode::encode('UTF-8', $title);
199
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
214
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
215
216
    my $deleted_record = $records[2];
217
    # Leader has the expected value (and record status "d")
218
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
200
};
219
};
201
220
202
subtest 'export without record_type' => sub {
221
subtest 'export without record_type' => sub {
203
- 

Return to bug 20551