View | Details | Raw Unified | Return to bug 35104
Collapse All | Expand All

(-)a/Koha/Biblio/Metadata.pm (-34 / +74 lines)
Lines 84-95 sub store { Link Here
84
84
85
            if ($stripped_marcxml) {
85
            if ($stripped_marcxml) {
86
86
87
                # Stripping fixed it – save the clean version and record all affected fields
87
                # Stripping fixed it – enumerate every bad character for the error log
88
                my @affected = _nonxml_affected_fields( $self->metadata );
88
                my @nonxml_chars  = _find_nonxml_chars( $self->metadata );
89
                my $field_context =
89
                my $field_context = do {
90
                    @affected
90
                    my %seen;
91
                    ? join( ', ', @affected )
91
                    join( ', ', grep { !$seen{$_}++ } map { $_->{field} } @nonxml_chars )
92
                    : 'unknown location';
92
                        || 'unknown location';
93
                };
93
94
94
                Koha::Logger->get->warn(
95
                Koha::Logger->get->warn(
95
                    sprintf(
96
                    sprintf(
Lines 98-104 sub store { Link Here
98
                    )
99
                    )
99
                );
100
                );
100
                $self->metadata($stripped_metadata);
101
                $self->metadata($stripped_metadata);
101
                $nonxml_error_message = "Non-XML characters stripped from: $field_context";
102
                $nonxml_error_message = \@nonxml_chars || $marcxml_error;
102
            } else {
103
            } else {
103
104
104
                # Truly unrecoverable
105
                # Truly unrecoverable
Lines 119-131 sub store { Link Here
119
    # Sync nonxml_stripped error rows: clear any existing, then re-add if needed
120
    # Sync nonxml_stripped error rows: clear any existing, then re-add if needed
120
    $self->metadata_errors->search( { error_type => 'nonxml_stripped' } )->delete;
121
    $self->metadata_errors->search( { error_type => 'nonxml_stripped' } )->delete;
121
    if ($nonxml_error_message) {
122
    if ($nonxml_error_message) {
122
        Koha::Biblio::Metadata::Error->new(
123
        my @occurrences =
123
            {
124
            ref $nonxml_error_message eq 'ARRAY'
124
                metadata_id => $self->id,
125
            ? @{$nonxml_error_message}
125
                error_type  => 'nonxml_stripped',
126
            : ();
126
                message     => $nonxml_error_message,
127
128
        if (@occurrences) {
129
            for my $occ (@occurrences) {
130
                Koha::Biblio::Metadata::Error->new(
131
                    {
132
                        metadata_id => $self->id,
133
                        error_type  => 'nonxml_stripped',
134
                        message     => sprintf(
135
                            "%s: invalid char value %d (U+%04X) at position %d",
136
                            $occ->{field}, $occ->{char_ord}, $occ->{char_ord}, $occ->{position}
137
                        ),
138
                    }
139
                )->store;
127
            }
140
            }
128
        )->store;
141
        } else {
142
143
            # Fallback: couldn't pinpoint individual chars – store the parser error
144
            Koha::Biblio::Metadata::Error->new(
145
                {
146
                    metadata_id => $self->id,
147
                    error_type  => 'nonxml_stripped',
148
                    message     => $nonxml_error_message,
149
                }
150
            )->store;
151
        }
129
    }
152
    }
130
153
131
    return $self;
154
    return $self;
Lines 360-409 sub _embed_items { Link Here
360
    return $record;
383
    return $record;
361
}
384
}
362
385
363
=head3 _nonxml_affected_fields
386
=head3 _find_nonxml_chars
387
388
    my @occurrences = _find_nonxml_chars( $marcxml_string );
389
390
Scans a raw MARC XML string for every individual character that is illegal
391
in XML 1.0.  Returns a list of hashrefs (one per character occurrence) with
392
the following keys:
393
394
=over 4
395
396
=item * C<field> - human-readable field reference, e.g. C<245$a> or C<001>
397
398
=item * C<char_ord> - ordinal (decimal) value of the bad character
364
399
365
    my @fields = _nonxml_affected_fields( $marcxml_string );
400
=item * C<position> - 1-based character offset within the field value
366
401
367
Scans a raw MARC XML string for every field and subfield that contains at
402
=back
368
least one character illegal in XML 1.0.  Returns a deduplicated list of
369
human-readable references such as C<245$a> (for subfields) or C<001> (for
370
control fields), in document order.
371
403
372
The same character class is used as L<C4::Charset/StripNonXmlChars>, so the
404
The same character class is used as L<C4::Charset/StripNonXmlChars>, so
373
set of affected fields reported here exactly matches what would be stripped.
405
every occurrence returned here is exactly one that would be stripped.
374
406
375
=cut
407
=cut
376
408
377
sub _nonxml_affected_fields {
409
sub _find_nonxml_chars {
378
    my ($marcxml) = @_;
410
    my ($marcxml) = @_;
379
411
380
    # Characters illegal in XML 1.0 – identical to the set stripped by StripNonXmlChars
412
    # Characters illegal in XML 1.0 – identical to the set stripped by StripNonXmlChars
381
    my $non_xml_re = qr/[^\x09\x0A\x0D\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]/;
413
    my $non_xml_re = qr/[^\x09\x0A\x0D\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]/;
382
414
383
    my @affected;
415
    my @occurrences;
384
    my %seen;
416
417
    my $scan = sub {
418
        my ( $field_ref, $val ) = @_;
419
        my $pos = 1;
420
        for my $char ( split //, $val ) {
421
            if ( $char =~ $non_xml_re ) {
422
                push @occurrences,
423
                    {
424
                    field    => $field_ref,
425
                    char_ord => ord($char),
426
                    position => $pos,
427
                    };
428
            }
429
            $pos++;
430
        }
431
    };
385
432
386
    # Scan every subfield inside every datafield
433
    # Scan every subfield inside every datafield
387
    while ( $marcxml =~ m{<datafield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</datafield>}gs ) {
434
    while ( $marcxml =~ m{<datafield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</datafield>}gs ) {
388
        my ( $tag, $content ) = ( $1, $2 );
435
        my ( $tag, $content ) = ( $1, $2 );
389
        while ( $content =~ m{<subfield\b[^>]*\bcode="(\w)"[^>]*>(.*?)</subfield>}gs ) {
436
        while ( $content =~ m{<subfield\b[^>]*\bcode="(\w)"[^>]*>(.*?)</subfield>}gs ) {
390
            my ( $code, $val ) = ( $1, $2 );
437
            $scan->( "$tag\$$1", $2 );
391
            my $ref = "$tag\$$code";
392
            if ( $val =~ $non_xml_re && !$seen{$ref}++ ) {
393
                push @affected, $ref;
394
            }
395
        }
438
        }
396
    }
439
    }
397
440
398
    # Scan control fields
441
    # Scan control fields
399
    while ( $marcxml =~ m{<controlfield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</controlfield>}gs ) {
442
    while ( $marcxml =~ m{<controlfield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</controlfield>}gs ) {
400
        my ( $tag, $val ) = ( $1, $2 );
443
        $scan->( $1, $2 );
401
        if ( $val =~ $non_xml_re && !$seen{$tag}++ ) {
402
            push @affected, $tag;
403
        }
404
    }
444
    }
405
445
406
    return @affected;
446
    return @occurrences;
407
}
447
}
408
448
409
=head3 _type
449
=head3 _type
(-)a/catalogue/MARCdetail.pl (-2 / +5 lines)
Lines 121-128 if ( $query->cookie("searchToOrder") ) { Link Here
121
}
121
}
122
122
123
$template->param( ocoins => $biblio_object->get_coins );
123
$template->param( ocoins => $biblio_object->get_coins );
124
my $nonxml_error = $biblio_object->metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
124
my $nonxml_errors = $biblio_object->metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } );
125
$template->param( nonxml_stripped => $nonxml_error ? $nonxml_error->message : undef );
125
if ( $nonxml_errors->count ) {
126
    my @messages = map { $_->message } $nonxml_errors->as_list;
127
    $template->param( nonxml_stripped => join( "\n", @messages ) );
128
}
126
129
127
#count of item linked
130
#count of item linked
128
my $itemcount = $biblio_object->items->count;
131
my $itemcount = $biblio_object->items->count;
(-)a/catalogue/detail.pl (-2 / +5 lines)
Lines 177-184 my $marcflavour = C4::Context->preference("marcflavour"); Link Here
177
$template->param( 'SpineLabelShowPrintOnBibDetails' => C4::Context->preference("SpineLabelShowPrintOnBibDetails") );
177
$template->param( 'SpineLabelShowPrintOnBibDetails' => C4::Context->preference("SpineLabelShowPrintOnBibDetails") );
178
178
179
$template->param( ocoins => !$invalid_marc_record ? $biblio->get_coins : undef );
179
$template->param( ocoins => !$invalid_marc_record ? $biblio->get_coins : undef );
180
my $nonxml_error = $biblio->metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
180
my $nonxml_errors = $biblio->metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } );
181
$template->param( nonxml_stripped => $nonxml_error ? $nonxml_error->message : undef );
181
if ( $nonxml_errors->count ) {
182
    my @messages = map { $_->message } $nonxml_errors->as_list;
183
    $template->param( nonxml_stripped => join( "\n", @messages ) );
184
}
182
185
183
# some useful variables for enhanced content;
186
# some useful variables for enhanced content;
184
# in each case, we're grabbing the first value we find in
187
# in each case, we're grabbing the first value we find in
(-)a/t/db_dependent/Biblio.t (-7 / +8 lines)
Lines 1205-1211 subtest 'GetFrameworkCode' => sub { Link Here
1205
};
1205
};
1206
1206
1207
subtest 'ModBiblio on record with strippable non-XML characters' => sub {
1207
subtest 'ModBiblio on record with strippable non-XML characters' => sub {
1208
    plan tests => 3;
1208
    plan tests => 4;
1209
1209
1210
    t::lib::Mocks::mock_preference( "CataloguingLog", 1 );
1210
    t::lib::Mocks::mock_preference( "CataloguingLog", 1 );
1211
1211
Lines 1221-1235 subtest 'ModBiblio on record with strippable non-XML characters' => sub { Link Here
1221
    my @warnings;
1221
    my @warnings;
1222
    C4::Biblio::ModBiblio( $record, $biblionumber, '', { warnings => \@warnings } );
1222
    C4::Biblio::ModBiblio( $record, $biblionumber, '', { warnings => \@warnings } );
1223
    like(
1223
    like(
1224
        $warnings[0]{message}, qr/Non-XML characters stripped from: /,
1224
        $warnings[0]{message}, qr/650\$a: invalid char value 31 \(U\+001F\) at position \d+/,
1225
        'Non-XML character stripping reported via warnings arrayref'
1225
        'Non-XML character stripping reported via warnings arrayref with field and char context'
1226
    );
1226
    );
1227
1227
1228
    my $metadata     = Koha::Biblios->find($biblionumber)->metadata;
1228
    my $metadata      = Koha::Biblios->find($biblionumber)->metadata;
1229
    my $nonxml_error = $metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
1229
    my @nonxml_errors = $metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->as_list;
1230
    ok( scalar @nonxml_errors, 'nonxml_stripped errors persisted in biblio_metadata_errors after ModBiblio' );
1230
    like(
1231
    like(
1231
        $nonxml_error->message, qr/Non-XML characters stripped from: /,
1232
        $nonxml_errors[0]->message, qr/650\$a: invalid char value 31 \(U\+001F\) at position \d+/,
1232
        'nonxml_stripped error message persisted in biblio_metadata_errors after ModBiblio'
1233
        'error message identifies field, char value, and position'
1233
    );
1234
    );
1234
1235
1235
    my $action_logs =
1236
    my $action_logs =
(-)a/t/db_dependent/Koha/Biblio/Metadata.t (-6 / +5 lines)
Lines 107-118 EOX Link Here
107
107
108
        lives_ok { $record->store } 'MARCXML with strippable characters stores successfully';
108
        lives_ok { $record->store } 'MARCXML with strippable characters stores successfully';
109
109
110
        my $error = $record->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
110
        my @errors = $record->metadata_errors->search( { error_type => 'nonxml_stripped' } )->as_list;
111
        ok( $error, 'nonxml_stripped error recorded after stripping' );
111
        ok( scalar @errors, 'nonxml_stripped error(s) recorded after stripping' );
112
        like(
112
        like(
113
            $error->message,
113
            $errors[0]->message,
114
            qr/245\$a/,
114
            qr/245\$a: invalid char value 31 \(U\+001F\) at position \d+/,
115
            'error message identifies the affected MARC field and subfield'
115
            'error message identifies field, char value, and position'
116
        );
116
        );
117
117
118
        my $stored = Koha::Biblio::Metadatas->find( { biblionumber => $biblio->{biblionumber}, format => 'marcxml' } );
118
        my $stored = Koha::Biblio::Metadatas->find( { biblionumber => $biblio->{biblionumber}, format => 'marcxml' } );
119
- 

Return to bug 35104