View | Details | Raw Unified | Return to bug 35104
Collapse All | Expand All

(-)a/Koha/Biblio/Metadata.pm (-4 / +56 lines)
Lines 84-98 sub store { Link Here
84
84
85
            if ($stripped_marcxml) {
85
            if ($stripped_marcxml) {
86
86
87
                # Stripping fixed it – save the clean version
87
                # Stripping fixed it – save the clean version and record all affected fields
88
                my @affected = _nonxml_affected_fields( $self->metadata );
89
                my $field_context =
90
                    @affected
91
                    ? join( ', ', @affected )
92
                    : 'unknown location';
93
88
                Koha::Logger->get->warn(
94
                Koha::Logger->get->warn(
89
                    sprintf(
95
                    sprintf(
90
                        "Non-XML characters stripped from bibliographic record (biblionumber=%s): %s",
96
                        "Non-XML characters stripped from bibliographic record (biblionumber=%s) in %s: %s",
91
                        $self->biblionumber // 'N/A', $marcxml_error
97
                        $self->biblionumber // 'N/A', $field_context, $marcxml_error
92
                    )
98
                    )
93
                );
99
                );
94
                $self->metadata($stripped_metadata);
100
                $self->metadata($stripped_metadata);
95
                $nonxml_error_message = $marcxml_error;
101
                $nonxml_error_message = "Non-XML characters stripped from: $field_context";
96
            } else {
102
            } else {
97
103
98
                # Truly unrecoverable
104
                # Truly unrecoverable
Lines 354-359 sub _embed_items { Link Here
354
    return $record;
360
    return $record;
355
}
361
}
356
362
363
=head3 _nonxml_affected_fields
364
365
    my @fields = _nonxml_affected_fields( $marcxml_string );
366
367
Scans a raw MARC XML string for every field and subfield that contains at
368
least one character illegal in XML 1.0.  Returns a deduplicated list of
369
human-readable references such as C<245$a> (for subfields) or C<001> (for
370
control fields), in document order.
371
372
The same character class is used as L<C4::Charset/StripNonXmlChars>, so the
373
set of affected fields reported here exactly matches what would be stripped.
374
375
=cut
376
377
sub _nonxml_affected_fields {
378
    my ($marcxml) = @_;
379
380
    # Characters illegal in XML 1.0 – identical to the set stripped by StripNonXmlChars
381
    my $non_xml_re = qr/[^\x09\x0A\x0D\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]/;
382
383
    my @affected;
384
    my %seen;
385
386
    # Scan every subfield inside every datafield
387
    while ( $marcxml =~ m{<datafield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</datafield>}gs ) {
388
        my ( $tag, $content ) = ( $1, $2 );
389
        while ( $content =~ m{<subfield\b[^>]*\bcode="(\w)"[^>]*>(.*?)</subfield>}gs ) {
390
            my ( $code, $val ) = ( $1, $2 );
391
            my $ref = "$tag\$$code";
392
            if ( $val =~ $non_xml_re && !$seen{$ref}++ ) {
393
                push @affected, $ref;
394
            }
395
        }
396
    }
397
398
    # Scan control fields
399
    while ( $marcxml =~ m{<controlfield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</controlfield>}gs ) {
400
        my ( $tag, $val ) = ( $1, $2 );
401
        if ( $val =~ $non_xml_re && !$seen{$tag}++ ) {
402
            push @affected, $tag;
403
        }
404
    }
405
406
    return @affected;
407
}
408
357
=head3 _type
409
=head3 _type
358
410
359
=cut
411
=cut
(-)a/t/db_dependent/Biblio.t (-3 / +3 lines)
Lines 1221-1235 subtest 'ModBiblio on record with strippable non-XML characters' => sub { Link Here
1221
    my @warnings;
1221
    my @warnings;
1222
    C4::Biblio::ModBiblio( $record, $biblionumber, '', { warnings => \@warnings } );
1222
    C4::Biblio::ModBiblio( $record, $biblionumber, '', { warnings => \@warnings } );
1223
    like(
1223
    like(
1224
        $warnings[0]{message}, qr/parser error : PCDATA invalid Char value 31/,
1224
        $warnings[0]{message}, qr/Non-XML characters stripped from: /,
1225
        'Non-XML character stripping reported via warnings arrayref'
1225
        'Non-XML character stripping reported via warnings arrayref'
1226
    );
1226
    );
1227
1227
1228
    my $metadata     = Koha::Biblios->find($biblionumber)->metadata;
1228
    my $metadata     = Koha::Biblios->find($biblionumber)->metadata;
1229
    my $nonxml_error = $metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
1229
    my $nonxml_error = $metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
1230
    like(
1230
    like(
1231
        $nonxml_error->message, qr/parser error : PCDATA invalid Char value 31/,
1231
        $nonxml_error->message, qr/Non-XML characters stripped from: /,
1232
        'nonxml_stripped error persisted in biblio_metadata_errors after ModBiblio'
1232
        'nonxml_stripped error message persisted in biblio_metadata_errors after ModBiblio'
1233
    );
1233
    );
1234
1234
1235
    my $action_logs =
1235
    my $action_logs =
(-)a/t/db_dependent/Koha/Biblio/Metadata.t (-5 / +7 lines)
Lines 84-90 EOX Link Here
84
    };
84
    };
85
85
86
    subtest 'MARCXML with strippable non-XML characters' => sub {
86
    subtest 'MARCXML with strippable non-XML characters' => sub {
87
        plan tests => 3;
87
        plan tests => 4;
88
        $schema->storage->txn_begin;
88
        $schema->storage->txn_begin;
89
89
90
        # Build MARCXML containing a non-XML control character (chr(31) = Unit Separator)
90
        # Build MARCXML containing a non-XML control character (chr(31) = Unit Separator)
Lines 107-115 EOX Link Here
107
107
108
        lives_ok { $record->store } 'MARCXML with strippable characters stores successfully';
108
        lives_ok { $record->store } 'MARCXML with strippable characters stores successfully';
109
109
110
        ok(
110
        my $error = $record->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
111
            $record->metadata_errors->search( { error_type => 'nonxml_stripped' } )->count,
111
        ok( $error, 'nonxml_stripped error recorded after stripping' );
112
            'nonxml_stripped error recorded after stripping'
112
        like(
113
            $error->message,
114
            qr/245\$a/,
115
            'error message identifies the affected MARC field and subfield'
113
        );
116
        );
114
117
115
        my $stored = Koha::Biblio::Metadatas->find( { biblionumber => $biblio->{biblionumber}, format => 'marcxml' } );
118
        my $stored = Koha::Biblio::Metadatas->find( { biblionumber => $biblio->{biblionumber}, format => 'marcxml' } );
116
- 

Return to bug 35104