View | Details | Raw Unified | Return to bug 20551
Collapse All | Expand All

(-)a/Koha/Exporter/Record.pm (-28 / +97 lines)
Lines 6-15 use MARC::File::USMARC; Link Here
6
6
7
use C4::AuthoritiesMarc;
7
use C4::AuthoritiesMarc;
8
use C4::Biblio;
8
use C4::Biblio;
9
use C4::Charset;
9
use C4::Record;
10
use C4::Record;
10
use Koha::CsvProfiles;
11
use Koha::CsvProfiles;
12
use Koha::Database;
11
use Koha::Logger;
13
use Koha::Logger;
12
14
15
use MARC::Record;
16
use MARC::File::XML;
17
13
sub _get_record_for_export {
18
sub _get_record_for_export {
14
    my ($params)           = @_;
19
    my ($params)           = @_;
15
    my $record_type        = $params->{record_type};
20
    my $record_type        = $params->{record_type};
Lines 50-55 sub _get_record_for_export { Link Here
50
    return $record;
55
    return $record;
51
}
56
}
52
57
58
sub _get_deleted_biblio_for_export {
59
    my ($params)           = @_;
60
    my $record_id = $params->{record_id};
61
    # Creating schema is expensive, allow caller to
62
    # pass it so don't have to recreate for each call
63
    my $resultset = $params->{resultset} || Koha::Database
64
        ->new()
65
        ->schema()
66
        ->resultset('DeletedbiblioMetadata');
67
    my $marc_flavour = C4::Context->preference('marcflavour');
68
    my $biblio_metadata = $resultset->find({
69
        'biblionumber' => $record_id,
70
        'format' => 'marcxml',
71
        'marcflavour' => $marc_flavour
72
    });
73
    my $marc_xml = $biblio_metadata->metadata;
74
    $marc_xml = StripNonXmlChars($marc_xml);
75
76
    my $record = eval {
77
        MARC::Record::new_from_xml($marc_xml, 'UTF-8', $marc_flavour)
78
    };
79
    if ($@) {
80
        warn "Failed to load MARCXML for biblio with biblionumber \"$record_id\": $@";
81
        return undef;
82
    }
83
    # Set deleted flag (record status, position 05)
84
    my $leader = $record->leader;
85
    substr $leader, 5, 1, 'd';
86
    $record->leader($leader);
87
    return $record;
88
}
89
53
sub _get_authority_for_export {
90
sub _get_authority_for_export {
54
    my ($params) = @_;
91
    my ($params) = @_;
55
    my $authid = $params->{authid} || return;
92
    my $authid = $params->{authid} || return;
Lines 91-96 sub export { Link Here
91
128
92
    my $record_type        = $params->{record_type};
129
    my $record_type        = $params->{record_type};
93
    my $record_ids         = $params->{record_ids} || [];
130
    my $record_ids         = $params->{record_ids} || [];
131
    my $deleted_record_ids = $params->{deleted_record_ids} || [];
94
    my $format             = $params->{format};
132
    my $format             = $params->{format};
95
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
133
    my $itemnumbers        = $params->{itemnumbers} || [];    # Does not make sense with record_type eq auths
96
    my $export_items       = $params->{export_items};
134
    my $export_items       = $params->{export_items};
Lines 102-108 sub export { Link Here
102
        Koha::Logger->get->warn( "No record_type given." );
140
        Koha::Logger->get->warn( "No record_type given." );
103
        return;
141
        return;
104
    }
142
    }
105
    return unless @$record_ids;
143
    return unless (@{$record_ids} || @{$deleted_record_ids} && $format ne 'csv');
106
144
107
    my $fh;
145
    my $fh;
108
    if ( $output_filepath ) {
146
    if ( $output_filepath ) {
Lines 113-154 sub export { Link Here
113
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
151
        binmode STDOUT, ':encoding(UTF-8)' unless $format eq 'csv';
114
    }
152
    }
115
153
116
    if ( $format eq 'iso2709' ) {
154
    if ($format eq 'xml' || $format eq 'iso2709') {
117
        for my $record_id (@$record_ids) {
155
        my @records;
118
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
156
        @records = map {
119
            my $errorcount_on_decode = eval { scalar( MARC::File::USMARC->decode( $record->as_usmarc )->warnings() ) };
157
            my $record = _get_record_for_export({ %{$params}, record_id => $_ });
120
            if ( $errorcount_on_decode or $@ ) {
158
            Koha::Logger->get->warn( "Record $_ could not be loaded." ) unless $record;
121
                my $msg = "Record $record_id could not be exported. " .
159
            $record ? $record : ();
122
                    ( $@ // '' );
160
        } @{$record_ids};
123
                chomp $msg;
161
124
                Koha::Logger->get->info( $msg );
162
        my @deleted_records;
125
                next;
163
        if (@{$deleted_record_ids}) {
126
            }
164
            my $resultset = Koha::Database
127
            print $record->as_usmarc();
165
            ->new()
166
            ->schema()
167
            ->resultset('DeletedbiblioMetadata');
168
            @deleted_records = map {
169
                my $record = _get_deleted_biblio_for_export({
170
                    record_id => $_,
171
                    resultset => $resultset,
172
                });
173
                Koha::Logger->get->warn( "Deleted record $_ could not be loaded." ) unless $record;
174
                $record ? $record : ();
175
            } @{$deleted_record_ids};
128
        }
176
        }
129
    } elsif ( $format eq 'xml' ) {
177
        if ( $format eq 'iso2709' ) {
130
        my $marcflavour = C4::Context->preference("marcflavour");
178
            my $encoding_validator = sub {
131
        MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
179
                my ($record_type) = @_;
132
180
                return sub {
133
        print MARC::File::XML::header();
181
                    my ($record) = @_;
134
        print "\n";
182
                    my $errorcount_on_decode = eval { scalar(MARC::File::USMARC->decode($record->as_usmarc)->warnings()) };
135
        for my $record_id (@$record_ids) {
183
                    if ($errorcount_on_decode || $@) {
136
            my $record = _get_record_for_export( { %$params, record_id => $record_id } );
184
                        my ($id_tag, $id_subfield) = GetMarcFromKohaField('biblio.biblionumber', '');
137
            if( !$record ) {
185
                        my $record_id = $record->subfield($id_tag, $id_subfield);
138
                Koha::Logger->get->info( "Record $record_id could not be exported." );
186
                        my $msg = "$record_type $record_id could not be USMARC decoded/encoded. " . ($@ // '');
139
                next;
187
                        chomp $msg;
188
                        Koha::Logger->get->warn($msg);
189
                        return 0;
190
                    }
191
                    return 1;
192
                }
193
            };
194
            my $validator = $encoding_validator->('Record');
195
            for my $record (grep { $validator->($_) } @records) {
196
                print $record->as_usmarc();
197
            }
198
            if (@deleted_records) {
199
                $validator = $encoding_validator->('Deleted record');
200
                for my $deleted_record (grep { $validator->($_) } @deleted_records) {
201
                    print $deleted_record->as_usmarc();
202
                }
140
            }
203
            }
141
            print MARC::File::XML::record($record);
204
        } elsif ( $format eq 'xml' ) {
205
            my $marcflavour = C4::Context->preference("marcflavour");
206
            MARC::File::XML->default_record_format( ( $marcflavour eq 'UNIMARC' && $record_type eq 'auths' ) ? 'UNIMARCAUTH' : $marcflavour );
207
            print MARC::File::XML::header();
208
            print "\n";
209
            for my $record (@records, @deleted_records) {
210
                print MARC::File::XML::record($record);
211
                print "\n";
212
            }
213
            print MARC::File::XML::footer();
142
            print "\n";
214
            print "\n";
143
        }
215
        }
144
        print MARC::File::XML::footer();
145
        print "\n";
146
    } elsif ( $format eq 'csv' ) {
216
    } elsif ( $format eq 'csv' ) {
147
        die 'There is no valid csv profile defined for this export'
217
        die 'There is no valid csv profile defined for this export'
148
            unless Koha::CsvProfiles->find( $csv_profile_id );
218
            unless Koha::CsvProfiles->find( $csv_profile_id );
149
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
219
        print marc2csv( $record_ids, $csv_profile_id, $itemnumbers );
150
    }
220
    }
151
152
    close $fh if $output_filepath;
221
    close $fh if $output_filepath;
153
}
222
}
154
223
(-)a/misc/export_records.pl (-17 / +78 lines)
Lines 32-41 use Koha::CsvProfiles; Link Here
32
use Koha::Exporter::Record;
32
use Koha::Exporter::Record;
33
use Koha::DateUtils qw( dt_from_string output_pref );
33
use Koha::DateUtils qw( dt_from_string output_pref );
34
34
35
my ( $output_format, $timestamp, $dont_export_items, $csv_profile_id, $deleted_barcodes, $clean, $filename, $record_type, $id_list_file, $starting_authid, $ending_authid, $authtype, $starting_biblionumber, $ending_biblionumber, $itemtype, $starting_callnumber, $ending_callnumber, $start_accession, $end_accession, $help );
35
my (
36
    $output_format,
37
    $timestamp,
38
    $include_deleted,
39
    $deleted_only,
40
    $dont_export_items,
41
    $csv_profile_id,
42
    $deleted_barcodes,
43
    $clean,
44
    $filename,
45
    $record_type,
46
    $id_list_file,
47
    $starting_authid,
48
    $ending_authid,
49
    $authtype,
50
    $starting_biblionumber,
51
    $ending_biblionumber,
52
    $itemtype,
53
    $starting_callnumber,
54
    $ending_callnumber,
55
    $start_accession,
56
    $end_accession,
57
    $help
58
);
59
36
GetOptions(
60
GetOptions(
37
    'format=s'                => \$output_format,
61
    'format=s'                => \$output_format,
38
    'date=s'                  => \$timestamp,
62
    'date=s'                  => \$timestamp,
63
    'include_deleted'         => \$include_deleted,
64
    'deleted_only'            => \$deleted_only,
39
    'dont_export_items'       => \$dont_export_items,
65
    'dont_export_items'       => \$dont_export_items,
40
    'csv_profile_id=s'        => \$csv_profile_id,
66
    'csv_profile_id=s'        => \$csv_profile_id,
41
    'deleted_barcodes'        => \$deleted_barcodes,
67
    'deleted_barcodes'        => \$deleted_barcodes,
Lines 67-72 $record_type ||= 'bibs'; Link Here
67
# Retrocompatibility for the format parameter
93
# Retrocompatibility for the format parameter
68
$output_format = 'iso2709' if $output_format eq 'marc';
94
$output_format = 'iso2709' if $output_format eq 'marc';
69
95
96
if ($include_deleted || $deleted_only) {
97
   if ($record_type ne 'bibs') {
98
        pod2usage(q|Option "--include_deleted" or "--deleted_only" can only be used with "--record-type=bibs"|);
99
    }
100
    if (!$timestamp) {
101
        pod2usage(q|Option "--include_deleted" or "--deleted_only" requires that "--date" is also set|);
102
    }
103
    if ($output_format eq 'csv') {
104
        pod2usage(q|Option "--include_deleted" or "--deleted_only" cannot be used with "--format=csv"|);
105
    }
106
}
107
70
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
108
if ( $output_format eq 'csv' and $record_type eq 'auths' ) {
71
    pod2usage(q|CSV output is only available for biblio records|);
109
    pod2usage(q|CSV output is only available for biblio records|);
72
}
110
}
Lines 97-122 open STDOUT, '>', $filename if $filename; Link Here
97
135
98
136
99
my @record_ids;
137
my @record_ids;
138
my @deleted_record_ids;
100
139
101
$timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): '';
140
$timestamp = ($timestamp) ? output_pref({ dt => dt_from_string($timestamp), dateformat => 'iso', dateonly => 0, }): '';
102
141
103
if ( $record_type eq 'bibs' ) {
142
if ( $record_type eq 'bibs' ) {
104
    if ( $timestamp ) {
143
    if ( $timestamp ) {
105
        push @record_ids, $_->{biblionumber} for @{
144
        unless ($deleted_only) {
106
            $dbh->selectall_arrayref(q| (
145
            push @record_ids, $_->{biblionumber} for @{
107
                SELECT biblio_metadata.biblionumber
146
                $dbh->selectall_arrayref(q| (
108
                FROM biblio_metadata
147
                    SELECT biblio_metadata.biblionumber
109
                  LEFT JOIN items USING(biblionumber)
148
                    FROM biblio_metadata
110
                WHERE biblio_metadata.timestamp >= ?
149
                      LEFT JOIN items USING(biblionumber)
111
                  OR items.timestamp >= ?
150
                    WHERE biblio_metadata.timestamp >= ?
112
            ) UNION (
151
                      OR items.timestamp >= ?
113
                SELECT biblio_metadata.biblionumber
152
                ) UNION (
114
                FROM biblio_metadata
153
                    SELECT biblio_metadata.biblionumber
115
                  LEFT JOIN deleteditems USING(biblionumber)
154
                    FROM biblio_metadata
116
                WHERE biblio_metadata.timestamp >= ?
155
                      LEFT JOIN deleteditems USING(biblionumber)
117
                  OR deleteditems.timestamp >= ?
156
                    WHERE biblio_metadata.timestamp >= ?
118
            ) |, { Slice => {} }, ( $timestamp ) x 4 );
157
                      OR deleteditems.timestamp >= ?
119
        };
158
                ) |, { Slice => {} }, ( $timestamp ) x 4 );
159
            };
160
        }
161
        if ($include_deleted || $deleted_only) {
162
            push @deleted_record_ids, $_->{biblionumber} for @{
163
                $dbh->selectall_arrayref(q|
164
                    SELECT `biblionumber`
165
                    FROM `deletedbiblio`
166
                    WHERE `timestamp` >= ?
167
                |, { Slice => {} }, $timestamp);
168
            };
169
        }
120
    } else {
170
    } else {
121
        my $conditions = {
171
        my $conditions = {
122
            ( $starting_biblionumber or $ending_biblionumber )
172
            ( $starting_biblionumber or $ending_biblionumber )
Lines 199-204 else { Link Here
199
    Koha::Exporter::Record::export(
249
    Koha::Exporter::Record::export(
200
        {   record_type        => $record_type,
250
        {   record_type        => $record_type,
201
            record_ids         => \@record_ids,
251
            record_ids         => \@record_ids,
252
            deleted_record_ids => \@deleted_record_ids,
202
            format             => $output_format,
253
            format             => $output_format,
203
            csv_profile_id     => $csv_profile_id,
254
            csv_profile_id     => $csv_profile_id,
204
            export_items       => (not $dont_export_items),
255
            export_items       => (not $dont_export_items),
Lines 215-221 export records - This script exports record (biblios or authorities) Link Here
215
266
216
=head1 SYNOPSIS
267
=head1 SYNOPSIS
217
268
218
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
269
export_records.pl [-h|--help] [--format=format] [--date=datetime] [--include_deleted] [--deleted_only] [--record-type=TYPE] [--dont_export_items] [--deleted_barcodes] [--clean] [--id_list_file=PATH] --filename=outputfile
219
270
220
=head1 OPTIONS
271
=head1 OPTIONS
221
272
Lines 236-241 Print a brief help message. Link Here
236
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
287
                        mm/dd/yyyy[ hh:mm:ss] for us) records exported are the ones that
237
                        have been modified since DATETIME.
288
                        have been modified since DATETIME.
238
289
290
=item B<--include_deleted>
291
292
 --include_deleted      If enabled, when using --date option, deleted records will be included in export as marc records
293
                        with leader record status set to "d" (deleted).
294
295
=item B<--include_deleted>
296
297
 --include_deleted      If enabled, when using --date option, only deleted records will be included in export as marc
298
                        records with leader record status set to "d" (deleted).
299
239
=item B<--record-type>
300
=item B<--record-type>
240
301
241
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
302
 --record-type=TYPE     TYPE is 'bibs' or 'auths'.
(-)a/t/db_dependent/Exporter/Record.t (-13 / +31 lines)
Lines 58-63 $biblio_2->append_fields( Link Here
58
);
58
);
59
my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, '');
59
my ($biblionumber_2, $biblioitemnumber_2) = AddBiblio($biblio_2, '');
60
60
61
my $deleted_biblio = MARC::Record->new();
62
$deleted_biblio->leader('00136nam a22000617a 4500');
63
$deleted_biblio->append_fields(
64
    MARC::Field->new('100', ' ', ' ', a => 'Chopra, Deepak'),
65
    MARC::Field->new('245', ' ', ' ', a => 'The seven spiritual laws of success'),
66
);
67
my ($deleted_biblionumber) = AddBiblio($deleted_biblio, '');
68
DelBiblio($deleted_biblionumber);
69
61
my $bad_biblio = Koha::Biblio->new()->store();
70
my $bad_biblio = Koha::Biblio->new()->store();
62
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', marcflavour => C4::Context->preference('marcflavour') } )->store();
71
Koha::Biblio::Metadata->new( { biblionumber => $bad_biblio->id, format => 'marcxml', metadata => 'something wrong', marcflavour => C4::Context->preference('marcflavour') } )->store();
63
my $bad_biblionumber = $bad_biblio->id;
72
my $bad_biblionumber = $bad_biblio->id;
Lines 141-154 EOF Link Here
141
};
150
};
142
151
143
subtest 'export xml' => sub {
152
subtest 'export xml' => sub {
144
    plan tests => 3;
153
    plan tests => 4;
145
    my $generated_xml_file = '/tmp/test_export.xml';
154
    my $generated_xml_file = '/tmp/test_export.xml';
146
    warning_like {
155
    warning_like {
147
        Koha::Exporter::Record::export(
156
        Koha::Exporter::Record::export(
148
            {   record_type     => 'bibs',
157
            {   record_type        => 'bibs',
149
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
158
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
150
                format          => 'xml',
159
                deleted_record_ids => [ $deleted_biblionumber ],
151
                output_filepath => $generated_xml_file,
160
                format             => 'xml',
161
                output_filepath    => $generated_xml_file,
152
            }
162
            }
153
        );
163
        );
154
    }
164
    }
Lines 165-187 subtest 'export xml' => sub { Link Here
165
    while ( my $record = $records->next ) {
175
    while ( my $record = $records->next ) {
166
        push @records, $record;
176
        push @records, $record;
167
    }
177
    }
168
    is( scalar( @records ), 2, 'Export XML: 2 records should have been exported' );
178
    is( scalar( @records ), 3, 'Export XML: 3 records should have been exported' );
169
    my $second_record = $records[1];
179
    my $second_record = $records[1];
170
    my $title = $second_record->subfield(245, 'a');
180
    my $title = $second_record->subfield(245, 'a');
171
    $title = Encode::encode('UTF-8', $title);
181
    $title = Encode::encode('UTF-8', $title);
172
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
182
    is( $title, $biblio_2_title, 'Export XML: The title is correctly encoded' );
183
184
    my $deleted_record = $records[2];
185
    # Leader has the expected value (and record status "d")
186
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
173
};
187
};
174
188
175
subtest 'export iso2709' => sub {
189
subtest 'export iso2709' => sub {
176
    plan tests => 3;
190
    plan tests => 4;
177
    my $generated_mrc_file = '/tmp/test_export.mrc';
191
    my $generated_mrc_file = '/tmp/test_export.mrc';
178
    # Get all item infos
192
    # Get all item infos
179
    warning_like {
193
    warning_like {
180
        Koha::Exporter::Record::export(
194
        Koha::Exporter::Record::export(
181
            {   record_type     => 'bibs',
195
            {   record_type        => 'bibs',
182
                record_ids      => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
196
                record_ids         => [ $biblionumber_1, $bad_biblionumber, $biblionumber_2 ],
183
                format          => 'iso2709',
197
                deleted_record_ids => [ $deleted_biblionumber ],
184
                output_filepath => $generated_mrc_file,
198
                format             => 'iso2709',
199
                output_filepath    => $generated_mrc_file,
185
            }
200
            }
186
        );
201
        );
187
    }
202
    }
Lines 192-202 subtest 'export iso2709' => sub { Link Here
192
    while ( my $record = $records->next ) {
207
    while ( my $record = $records->next ) {
193
        push @records, $record;
208
        push @records, $record;
194
    }
209
    }
195
    is( scalar( @records ), 2, 'Export ISO2709: 2 records should have been exported' );
210
    is( scalar( @records ), 3, 'Export ISO2709: 3 records should have been exported' );
196
    my $second_record = $records[1];
211
    my $second_record = $records[1];
197
    my $title = $second_record->subfield(245, 'a');
212
    my $title = $second_record->subfield(245, 'a');
198
    $title = Encode::encode('UTF-8', $title);
213
    $title = Encode::encode('UTF-8', $title);
199
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
214
    is( $title, $biblio_2_title, 'Export ISO2709: The title is correctly encoded' );
215
216
    my $deleted_record = $records[2];
217
    # Leader has the expected value (and record status "d")
218
    is( $deleted_record->leader, '00136dam a22000617a 4500', 'Deleted record has the correct leader value' );
200
};
219
};
201
220
202
subtest 'export without record_type' => sub {
221
subtest 'export without record_type' => sub {
203
- 

Return to bug 20551