|
Lines 84-95
sub store {
Link Here
|
| 84 |
|
84 |
|
| 85 |
if ($stripped_marcxml) { |
85 |
if ($stripped_marcxml) { |
| 86 |
|
86 |
|
| 87 |
# Stripping fixed it – save the clean version and record all affected fields |
87 |
# Stripping fixed it – enumerate every bad character for the error log |
| 88 |
my @affected = _nonxml_affected_fields( $self->metadata ); |
88 |
my @nonxml_chars = _find_nonxml_chars( $self->metadata ); |
| 89 |
my $field_context = |
89 |
my $field_context = do { |
| 90 |
@affected |
90 |
my %seen; |
| 91 |
? join( ', ', @affected ) |
91 |
join( ', ', grep { !$seen{$_}++ } map { $_->{field} } @nonxml_chars ) |
| 92 |
: 'unknown location'; |
92 |
|| 'unknown location'; |
|
|
93 |
}; |
| 93 |
|
94 |
|
| 94 |
Koha::Logger->get->warn( |
95 |
Koha::Logger->get->warn( |
| 95 |
sprintf( |
96 |
sprintf( |
|
Lines 98-104
sub store {
Link Here
|
| 98 |
) |
99 |
) |
| 99 |
); |
100 |
); |
| 100 |
$self->metadata($stripped_metadata); |
101 |
$self->metadata($stripped_metadata); |
| 101 |
$nonxml_error_message = "Non-XML characters stripped from: $field_context"; |
102 |
$nonxml_error_message = \@nonxml_chars || $marcxml_error; |
| 102 |
} else { |
103 |
} else { |
| 103 |
|
104 |
|
| 104 |
# Truly unrecoverable |
105 |
# Truly unrecoverable |
|
Lines 119-131
sub store {
Link Here
|
| 119 |
# Sync nonxml_stripped error rows: clear any existing, then re-add if needed |
120 |
# Sync nonxml_stripped error rows: clear any existing, then re-add if needed |
| 120 |
$self->metadata_errors->search( { error_type => 'nonxml_stripped' } )->delete; |
121 |
$self->metadata_errors->search( { error_type => 'nonxml_stripped' } )->delete; |
| 121 |
if ($nonxml_error_message) { |
122 |
if ($nonxml_error_message) { |
| 122 |
Koha::Biblio::Metadata::Error->new( |
123 |
my @occurrences = |
| 123 |
{ |
124 |
ref $nonxml_error_message eq 'ARRAY' |
| 124 |
metadata_id => $self->id, |
125 |
? @{$nonxml_error_message} |
| 125 |
error_type => 'nonxml_stripped', |
126 |
: (); |
| 126 |
message => $nonxml_error_message, |
127 |
|
|
|
128 |
if (@occurrences) { |
| 129 |
for my $occ (@occurrences) { |
| 130 |
Koha::Biblio::Metadata::Error->new( |
| 131 |
{ |
| 132 |
metadata_id => $self->id, |
| 133 |
error_type => 'nonxml_stripped', |
| 134 |
message => sprintf( |
| 135 |
"%s: invalid char value %d (U+%04X) at position %d", |
| 136 |
$occ->{field}, $occ->{char_ord}, $occ->{char_ord}, $occ->{position} |
| 137 |
), |
| 138 |
} |
| 139 |
)->store; |
| 127 |
} |
140 |
} |
| 128 |
)->store; |
141 |
} else { |
|
|
142 |
|
| 143 |
# Fallback: couldn't pinpoint individual chars – store the parser error |
| 144 |
Koha::Biblio::Metadata::Error->new( |
| 145 |
{ |
| 146 |
metadata_id => $self->id, |
| 147 |
error_type => 'nonxml_stripped', |
| 148 |
message => $nonxml_error_message, |
| 149 |
} |
| 150 |
)->store; |
| 151 |
} |
| 129 |
} |
152 |
} |
| 130 |
|
153 |
|
| 131 |
return $self; |
154 |
return $self; |
|
Lines 360-409
sub _embed_items {
Link Here
|
| 360 |
return $record; |
383 |
return $record; |
| 361 |
} |
384 |
} |
| 362 |
|
385 |
|
| 363 |
=head3 _nonxml_affected_fields |
386 |
=head3 _find_nonxml_chars |
|
|
387 |
|
| 388 |
my @occurrences = _find_nonxml_chars( $marcxml_string ); |
| 389 |
|
| 390 |
Scans a raw MARC XML string for every individual character that is illegal |
| 391 |
in XML 1.0. Returns a list of hashrefs (one per character occurrence) with |
| 392 |
the following keys: |
| 393 |
|
| 394 |
=over 4 |
| 395 |
|
| 396 |
=item * C<field> - human-readable field reference, e.g. C<245$a> or C<001> |
| 397 |
|
| 398 |
=item * C<char_ord> - ordinal (decimal) value of the bad character |
| 364 |
|
399 |
|
| 365 |
my @fields = _nonxml_affected_fields( $marcxml_string ); |
400 |
=item * C<position> - 1-based character offset within the field value |
| 366 |
|
401 |
|
| 367 |
Scans a raw MARC XML string for every field and subfield that contains at |
402 |
=back |
| 368 |
least one character illegal in XML 1.0. Returns a deduplicated list of |
|
|
| 369 |
human-readable references such as C<245$a> (for subfields) or C<001> (for |
| 370 |
control fields), in document order. |
| 371 |
|
403 |
|
| 372 |
The same character class is used as L<C4::Charset/StripNonXmlChars>, so the |
404 |
The same character class is used as L<C4::Charset/StripNonXmlChars>, so |
| 373 |
set of affected fields reported here exactly matches what would be stripped. |
405 |
every occurrence returned here is exactly one that would be stripped. |
| 374 |
|
406 |
|
| 375 |
=cut |
407 |
=cut |
| 376 |
|
408 |
|
| 377 |
sub _nonxml_affected_fields { |
409 |
sub _find_nonxml_chars { |
| 378 |
my ($marcxml) = @_; |
410 |
my ($marcxml) = @_; |
| 379 |
|
411 |
|
| 380 |
# Characters illegal in XML 1.0 – identical to the set stripped by StripNonXmlChars |
412 |
# Characters illegal in XML 1.0 – identical to the set stripped by StripNonXmlChars |
| 381 |
my $non_xml_re = qr/[^\x09\x0A\x0D\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]/; |
413 |
my $non_xml_re = qr/[^\x09\x0A\x0D\x{0020}-\x{D7FF}\x{E000}-\x{FFFD}\x{10000}-\x{10FFFF}]/; |
| 382 |
|
414 |
|
| 383 |
my @affected; |
415 |
my @occurrences; |
| 384 |
my %seen; |
416 |
|
|
|
417 |
my $scan = sub { |
| 418 |
my ( $field_ref, $val ) = @_; |
| 419 |
my $pos = 1; |
| 420 |
for my $char ( split //, $val ) { |
| 421 |
if ( $char =~ $non_xml_re ) { |
| 422 |
push @occurrences, |
| 423 |
{ |
| 424 |
field => $field_ref, |
| 425 |
char_ord => ord($char), |
| 426 |
position => $pos, |
| 427 |
}; |
| 428 |
} |
| 429 |
$pos++; |
| 430 |
} |
| 431 |
}; |
| 385 |
|
432 |
|
| 386 |
# Scan every subfield inside every datafield |
433 |
# Scan every subfield inside every datafield |
| 387 |
while ( $marcxml =~ m{<datafield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</datafield>}gs ) { |
434 |
while ( $marcxml =~ m{<datafield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</datafield>}gs ) { |
| 388 |
my ( $tag, $content ) = ( $1, $2 ); |
435 |
my ( $tag, $content ) = ( $1, $2 ); |
| 389 |
while ( $content =~ m{<subfield\b[^>]*\bcode="(\w)"[^>]*>(.*?)</subfield>}gs ) { |
436 |
while ( $content =~ m{<subfield\b[^>]*\bcode="(\w)"[^>]*>(.*?)</subfield>}gs ) { |
| 390 |
my ( $code, $val ) = ( $1, $2 ); |
437 |
$scan->( "$tag\$$1", $2 ); |
| 391 |
my $ref = "$tag\$$code"; |
|
|
| 392 |
if ( $val =~ $non_xml_re && !$seen{$ref}++ ) { |
| 393 |
push @affected, $ref; |
| 394 |
} |
| 395 |
} |
438 |
} |
| 396 |
} |
439 |
} |
| 397 |
|
440 |
|
| 398 |
# Scan control fields |
441 |
# Scan control fields |
| 399 |
while ( $marcxml =~ m{<controlfield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</controlfield>}gs ) { |
442 |
while ( $marcxml =~ m{<controlfield\b[^>]*\btag="(\w+)"[^>]*>(.*?)</controlfield>}gs ) { |
| 400 |
my ( $tag, $val ) = ( $1, $2 ); |
443 |
$scan->( $1, $2 ); |
| 401 |
if ( $val =~ $non_xml_re && !$seen{$tag}++ ) { |
|
|
| 402 |
push @affected, $tag; |
| 403 |
} |
| 404 |
} |
444 |
} |
| 405 |
|
445 |
|
| 406 |
return @affected; |
446 |
return @occurrences; |
| 407 |
} |
447 |
} |
| 408 |
|
448 |
|
| 409 |
=head3 _type |
449 |
=head3 _type |