View | Details | Raw Unified | Return to bug 35104
Collapse All | Expand All

(-)a/C4/Biblio.pm (-3 / +6 lines)
Lines 3004-3012 sub ModBiblioMarc { Link Here
3004
        }
3004
        }
3005
    )->store;
3005
    )->store;
3006
3006
3007
    if ( $m_rs->nonxml_stripped && ref $options->{warnings} eq 'ARRAY' ) {
3007
    if ( ref $options->{warnings} eq 'ARRAY' ) {
3008
        push @{ $options->{warnings} },
3008
        my $nonxml_error = $m_rs->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
3009
            { type => 'nonxml_stripped', message => $m_rs->nonxml_stripped };
3009
        if ($nonxml_error) {
3010
            push @{ $options->{warnings} },
3011
                { type => 'nonxml_stripped', message => $nonxml_error->message };
3012
        }
3010
    }
3013
    }
3011
3014
3012
    unless ($skip_record_index) {
3015
    unless ($skip_record_index) {
(-)a/Koha/Biblio/Metadata.pm (-9 / +40 lines)
Lines 24-29 use C4::Biblio qw( GetMarcFromKohaField ); Link Here
24
use C4::Charset qw( StripNonXmlChars );
24
use C4::Charset qw( StripNonXmlChars );
25
use C4::Items   qw( GetMarcItem );
25
use C4::Items   qw( GetMarcItem );
26
26
27
use Koha::Biblio::Metadata::Error;
28
use Koha::Biblio::Metadata::Errors;
27
use Koha::Database;
29
use Koha::Database;
28
use Koha::Exceptions::Metadata;
30
use Koha::Exceptions::Metadata;
29
use Koha::Logger;
31
use Koha::Logger;
Lines 47-54 Metadata specific store method to catch errant characters prior Link Here
47
to committing to the database.
49
to committing to the database.
48
50
49
If the MARCXML cannot be parsed but can be recovered by stripping non-XML
51
If the MARCXML cannot be parsed but can be recovered by stripping non-XML
50
characters (C<StripNonXmlChars>), the stripped version is saved and
52
characters (C<StripNonXmlChars>), the stripped version is saved and a
51
C<nonxml_stripped> is persisted on the row so callers and the UI can notify the user.
53
C<nonxml_stripped> error row is written to C<biblio_metadata_errors> so
54
callers and the UI can notify the user.  Any pre-existing C<nonxml_stripped>
55
error is removed when the record parses cleanly.
52
56
53
If the MARCXML cannot be recovered at all, a
57
If the MARCXML cannot be recovered at all, a
54
I<Koha::Exceptions::Metadata::Invalid> exception is thrown.
58
I<Koha::Exceptions::Metadata::Invalid> exception is thrown.
Lines 58-63 I<Koha::Exceptions::Metadata::Invalid> exception is thrown. Link Here
58
sub store {
62
sub store {
59
    my $self = shift;
63
    my $self = shift;
60
64
65
    my $nonxml_error_message;
66
61
    # Check marcxml will roundtrip
67
    # Check marcxml will roundtrip
62
    if ( $self->format eq 'marcxml' ) {
68
    if ( $self->format eq 'marcxml' ) {
63
69
Lines 69-79 sub store { Link Here
69
        };
75
        };
70
        my $marcxml_error = $@;
76
        my $marcxml_error = $@;
71
        chomp $marcxml_error;
77
        chomp $marcxml_error;
72
        if ($marcxml) {
73
78
74
            # Clean parse – clear any previously recorded stripping
79
        unless ($marcxml) {
75
            $self->nonxml_stripped(undef);
76
        } else {
77
80
78
            # Attempt recovery by stripping non-XML characters
81
            # Attempt recovery by stripping non-XML characters
79
            my $stripped_metadata = StripNonXmlChars( $self->metadata );
82
            my $stripped_metadata = StripNonXmlChars( $self->metadata );
Lines 81-87 sub store { Link Here
81
84
82
            if ($stripped_marcxml) {
85
            if ($stripped_marcxml) {
83
86
84
                # Stripping fixed it – save the clean version and record what happened
87
                # Stripping fixed it – save the clean version
85
                Koha::Logger->get->warn(
88
                Koha::Logger->get->warn(
86
                    sprintf(
89
                    sprintf(
87
                        "Non-XML characters stripped from bibliographic record (biblionumber=%s): %s",
90
                        "Non-XML characters stripped from bibliographic record (biblionumber=%s): %s",
Lines 89-95 sub store { Link Here
89
                    )
92
                    )
90
                );
93
                );
91
                $self->metadata($stripped_metadata);
94
                $self->metadata($stripped_metadata);
92
                $self->nonxml_stripped($marcxml_error);
95
                $nonxml_error_message = $marcxml_error;
93
            } else {
96
            } else {
94
97
95
                # Truly unrecoverable
98
                # Truly unrecoverable
Lines 105-111 sub store { Link Here
105
        }
108
        }
106
    }
109
    }
107
110
108
    return $self->SUPER::store;
111
    $self->SUPER::store;
112
113
    # Sync nonxml_stripped error rows: clear any existing, then re-add if needed
114
    $self->metadata_errors->search( { error_type => 'nonxml_stripped' } )->delete;
115
    if ($nonxml_error_message) {
116
        Koha::Biblio::Metadata::Error->new(
117
            {
118
                metadata_id => $self->id,
119
                error_type  => 'nonxml_stripped',
120
                message     => $nonxml_error_message,
121
            }
122
        )->store;
123
    }
124
125
    return $self;
126
}
127
128
=head3 metadata_errors
129
130
    my $errors = $metadata->metadata_errors;
131
132
Returns a I<Koha::Biblio::Metadata::Errors> resultset for errors associated
133
with this metadata record.
134
135
=cut
136
137
sub metadata_errors {
138
    my ($self) = @_;
139
    return Koha::Biblio::Metadata::Errors->_new_from_dbic( scalar $self->_result->biblio_metadata_errors );
109
}
140
}
110
141
111
=head3 record
142
=head3 record
(-)a/Koha/Biblio/Metadata/Error.pm (+38 lines)
Line 0 Link Here
1
package Koha::Biblio::Metadata::Error;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <https://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
20
use base qw(Koha::Object);
21
22
=head1 NAME
23
24
Koha::Biblio::Metadata::Error - Koha Biblio Metadata Error object class
25
26
=head1 API
27
28
=head2 Internal methods
29
30
=head3 _type
31
32
=cut
33
34
sub _type {
35
    return 'BiblioMetadataError';
36
}
37
38
1;
(-)a/Koha/Biblio/Metadata/Errors.pm (+48 lines)
Line 0 Link Here
1
package Koha::Biblio::Metadata::Errors;
2
3
# This file is part of Koha.
4
#
5
# Koha is free software; you can redistribute it and/or modify it
6
# under the terms of the GNU General Public License as published by
7
# the Free Software Foundation; either version 3 of the License, or
8
# (at your option) any later version.
9
#
10
# Koha is distributed in the hope that it will be useful, but
11
# WITHOUT ANY WARRANTY; without even the implied warranty of
12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
# GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License
16
# along with Koha; if not, see <https://www.gnu.org/licenses>.
17
18
use Modern::Perl;
19
20
use Koha::Biblio::Metadata::Error;
21
22
use base qw(Koha::Objects);
23
24
=head1 NAME
25
26
Koha::Biblio::Metadata::Errors - Koha Biblio Metadata Errors object set class
27
28
=head1 API
29
30
=head2 Internal methods
31
32
=head3 _type
33
34
=cut
35
36
sub _type {
37
    return 'BiblioMetadataError';
38
}
39
40
=head3 object_class
41
42
=cut
43
44
sub object_class {
45
    return 'Koha::Biblio::Metadata::Error';
46
}
47
48
1;
(-)a/catalogue/MARCdetail.pl (-2 / +3 lines)
Lines 120-127 if ( $query->cookie("searchToOrder") ) { Link Here
120
    );
120
    );
121
}
121
}
122
122
123
$template->param( ocoins          => $biblio_object->get_coins );
123
$template->param( ocoins => $biblio_object->get_coins );
124
$template->param( nonxml_stripped => $biblio_object->metadata->nonxml_stripped );
124
my $nonxml_error = $biblio_object->metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
125
$template->param( nonxml_stripped => $nonxml_error ? $nonxml_error->message : undef );
125
126
126
#count of item linked
127
#count of item linked
127
my $itemcount = $biblio_object->items->count;
128
my $itemcount = $biblio_object->items->count;
(-)a/catalogue/detail.pl (-2 / +3 lines)
Lines 176-183 my $marcflavour = C4::Context->preference("marcflavour"); Link Here
176
176
177
$template->param( 'SpineLabelShowPrintOnBibDetails' => C4::Context->preference("SpineLabelShowPrintOnBibDetails") );
177
$template->param( 'SpineLabelShowPrintOnBibDetails' => C4::Context->preference("SpineLabelShowPrintOnBibDetails") );
178
178
179
$template->param( ocoins          => !$invalid_marc_record ? $biblio->get_coins : undef );
179
$template->param( ocoins => !$invalid_marc_record ? $biblio->get_coins : undef );
180
$template->param( nonxml_stripped => $biblio->metadata->nonxml_stripped );
180
my $nonxml_error = $biblio->metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
181
$template->param( nonxml_stripped => $nonxml_error ? $nonxml_error->message : undef );
181
182
182
# some useful variables for enhanced content;
183
# some useful variables for enhanced content;
183
# in each case, we're grabbing the first value we find in
184
# in each case, we're grabbing the first value we find in
(-)a/installer/data/mysql/atomicupdate/bug_35104.pl (-6 / +23 lines)
Lines 3-23 use Koha::Installer::Output qw(say_warning say_success say_info); Link Here
3
3
4
return {
4
return {
5
    bug_number  => "35104",
5
    bug_number  => "35104",
6
    description => "Add nonxml_stripped column to biblio_metadata",
6
    description => "Add biblio_metadata_errors table for normalised error tracking",
7
    up          => sub {
7
    up          => sub {
8
        my ($args) = @_;
8
        my ($args) = @_;
9
        my ( $dbh, $out ) = @$args{qw(dbh out)};
9
        my ( $dbh, $out ) = @$args{qw(dbh out)};
10
10
11
        unless ( column_exists( 'biblio_metadata', 'nonxml_stripped' ) ) {
11
        # Remove the old nonxml_stripped column if it was added by an earlier version of this patch
12
        if ( column_exists( 'biblio_metadata', 'nonxml_stripped' ) ) {
12
            $dbh->do(
13
            $dbh->do(
13
                q{
14
                q{
14
                    ALTER TABLE biblio_metadata
15
                    ALTER TABLE biblio_metadata
15
                        ADD COLUMN `nonxml_stripped` mediumtext NULL DEFAULT NULL
16
                        DROP COLUMN `nonxml_stripped`
16
                        COMMENT 'parse error message recorded when non-XML characters were stripped on last store()'
17
                        AFTER metadata
18
                }
17
                }
19
            );
18
            );
20
            say $out "Added new column 'biblio_metadata.nonxml_stripped'";
19
            say $out "Dropped column 'biblio_metadata.nonxml_stripped'";
20
        }
21
22
        unless ( TableExists('biblio_metadata_errors') ) {
23
            $dbh->do(
24
                q{
25
                    CREATE TABLE `biblio_metadata_errors` (
26
                        `id` int(11) NOT NULL AUTO_INCREMENT,
27
                        `metadata_id` int(11) NOT NULL COMMENT 'FK to biblio_metadata.id',
28
                        `error_type` varchar(64) NOT NULL COMMENT 'Error type identifier, e.g. nonxml_stripped',
29
                        `message` mediumtext DEFAULT NULL COMMENT 'Error message details',
30
                        `created_on` timestamp NOT NULL DEFAULT current_timestamp(),
31
                        PRIMARY KEY (`id`),
32
                        KEY `biblio_metadata_errors_fk_1` (`metadata_id`),
33
                        CONSTRAINT `biblio_metadata_errors_fk_1` FOREIGN KEY (`metadata_id`) REFERENCES `biblio_metadata` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
34
                    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
35
                }
36
            );
37
            say $out "Added new table 'biblio_metadata_errors'";
21
        }
38
        }
22
    },
39
    },
23
};
40
};
(-)a/installer/data/mysql/kohastructure.sql (-1 / +19 lines)
Lines 1148-1154 CREATE TABLE `biblio_metadata` ( Link Here
1148
  `format` varchar(16) NOT NULL,
1148
  `format` varchar(16) NOT NULL,
1149
  `schema` varchar(16) NOT NULL,
1149
  `schema` varchar(16) NOT NULL,
1150
  `metadata` longtext NOT NULL,
1150
  `metadata` longtext NOT NULL,
1151
  `nonxml_stripped` mediumtext NULL DEFAULT NULL,
1152
  `timestamp` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
1151
  `timestamp` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(),
1153
  `record_source_id` int(11) DEFAULT NULL COMMENT 'The record source for the metadata',
1152
  `record_source_id` int(11) DEFAULT NULL COMMENT 'The record source for the metadata',
1154
  PRIMARY KEY (`id`),
1153
  PRIMARY KEY (`id`),
Lines 1160-1165 CREATE TABLE `biblio_metadata` ( Link Here
1160
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
1159
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
1161
/*!40101 SET character_set_client = @saved_cs_client */;
1160
/*!40101 SET character_set_client = @saved_cs_client */;
1162
1161
1162
--
1163
-- Table structure for table `biblio_metadata_errors`
1164
--
1165
1166
DROP TABLE IF EXISTS `biblio_metadata_errors`;
1167
/*!40101 SET @saved_cs_client     = @@character_set_client */;
1168
/*!40101 SET character_set_client = utf8mb4 */;
1169
CREATE TABLE `biblio_metadata_errors` (
1170
  `id` int(11) NOT NULL AUTO_INCREMENT,
1171
  `metadata_id` int(11) NOT NULL COMMENT 'FK to biblio_metadata.id',
1172
  `error_type` varchar(64) NOT NULL COMMENT 'Error type identifier, e.g. nonxml_stripped',
1173
  `message` mediumtext DEFAULT NULL COMMENT 'Error message details',
1174
  `created_on` timestamp NOT NULL DEFAULT current_timestamp(),
1175
  PRIMARY KEY (`id`),
1176
  KEY `biblio_metadata_errors_fk_1` (`metadata_id`),
1177
  CONSTRAINT `biblio_metadata_errors_fk_1` FOREIGN KEY (`metadata_id`) REFERENCES `biblio_metadata` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
1178
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
1179
/*!40101 SET character_set_client = @saved_cs_client */;
1180
1163
--
1181
--
1164
-- Table structure for table `biblioitems`
1182
-- Table structure for table `biblioitems`
1165
--
1183
--
(-)a/t/db_dependent/Biblio.t (-3 / +4 lines)
Lines 1225-1234 subtest 'ModBiblio on record with strippable non-XML characters' => sub { Link Here
1225
        'Non-XML character stripping reported via warnings arrayref'
1225
        'Non-XML character stripping reported via warnings arrayref'
1226
    );
1226
    );
1227
1227
1228
    my $metadata = Koha::Biblios->find($biblionumber)->metadata;
1228
    my $metadata     = Koha::Biblios->find($biblionumber)->metadata;
1229
    my $nonxml_error = $metadata->metadata_errors->search( { error_type => 'nonxml_stripped' } )->next;
1229
    like(
1230
    like(
1230
        $metadata->nonxml_stripped, qr/parser error : PCDATA invalid Char value 31/,
1231
        $nonxml_error->message, qr/parser error : PCDATA invalid Char value 31/,
1231
        'nonxml_stripped persisted on biblio_metadata after ModBiblio'
1232
        'nonxml_stripped error persisted in biblio_metadata_errors after ModBiblio'
1232
    );
1233
    );
1233
1234
1234
    my $action_logs =
1235
    my $action_logs =
(-)a/t/db_dependent/Koha/Biblio/Metadata.t (-3 / +8 lines)
Lines 75-81 EOX Link Here
75
75
76
        lives_ok { $record->store }
76
        lives_ok { $record->store }
77
        'Valid MARCXML record stores successfully';
77
        'Valid MARCXML record stores successfully';
78
        is( $record->nonxml_stripped, undef, 'nonxml_stripped is undef for clean record' );
78
        is(
79
            $record->metadata_errors->search( { error_type => 'nonxml_stripped' } )->count,
80
            0, 'No nonxml_stripped error for clean record'
81
        );
79
82
80
        $schema->storage->txn_rollback;
83
        $schema->storage->txn_rollback;
81
    };
84
    };
Lines 104-110 EOX Link Here
104
107
105
        lives_ok { $record->store } 'MARCXML with strippable characters stores successfully';
108
        lives_ok { $record->store } 'MARCXML with strippable characters stores successfully';
106
109
107
        ok( $record->nonxml_stripped, 'nonxml_stripped is set after stripping' );
110
        ok(
111
            $record->metadata_errors->search( { error_type => 'nonxml_stripped' } )->count,
112
            'nonxml_stripped error recorded after stripping'
113
        );
108
114
109
        my $stored = Koha::Biblio::Metadatas->find( { biblionumber => $biblio->{biblionumber}, format => 'marcxml' } );
115
        my $stored = Koha::Biblio::Metadatas->find( { biblionumber => $biblio->{biblionumber}, format => 'marcxml' } );
110
        is(
116
        is(
111
- 

Return to bug 35104