View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/Koha/OAI/Client/Record.pm (+257 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Record;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use XML::LibXSLT;
24
use MARC::Record;
25
use URI;
26
use File::Basename;
27
28
use C4::Context;
29
use C4::Biblio;
30
use C4::ImportBatch;
31
use C4::Matcher;
32
33
use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
34
35
sub new {
36
    my ($class, $args) = @_;
37
    $args = {} unless defined $args;
38
39
    if (my $inxml = $args->{xml_string}){
40
41
        #Parse the XML string into a XML::LibXML object
42
        my $doc = XML::LibXML->load_xml(string => $inxml, { no_blanks => 1 });
43
        #NOTE: Don't load blank nodes
44
        $args->{doc} = $doc;
45
46
        #Get the root element
47
        my $root = $doc->documentElement;
48
49
        #Register namespaces for searching purposes
50
        my $xpc = XML::LibXML::XPathContext->new();
51
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
52
53
        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
54
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
55
        $args->{header_identifier} = $identifier->textContent;
56
57
        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
58
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
59
        $args->{header_datestamp} = $datestamp->textContent;
60
61
        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
62
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
63
        $args->{header_status} = $status_node ? $status_node->textContent : "";
64
    }
65
66
    return bless ($args, $class);
67
}
68
69
sub is_deleted_upstream {
70
    my ($self, $args) = @_;
71
    if ($self->{header_status}){
72
        if ($self->{header_status} eq "deleted"){
73
            return 1;
74
        }
75
    }
76
    return 0;
77
}
78
79
sub set_filter {
80
    my ($self, $filter_definition) = @_;
81
82
    #Source a default XSLT to use for filtering
83
    my $htdocs  = C4::Context->config('intrahtdocs');
84
    my $theme   = C4::Context->preference("template");
85
    #FIXME: This doesn't work for UNIMARC, as it's specific to MARC21!
86
    $self->{filter} = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl";
87
88
    if ($filter_definition){
89
        my ($filter_type, $filter) = $self->_parse_filter($filter_definition);
90
        if ($filter_type eq "xslt"){
91
            if (  -f $filter ){
92
                $self->{filter} = $filter;
93
                $self->{filter_type} = "xslt";
94
            }
95
        }
96
    }
97
}
98
99
sub _parse_filter {
100
    my ($self,$filter_definition) = @_;
101
    my ($type,$filter);
102
    my $filter_uri = URI->new($filter_definition);
103
    if ($filter_uri){
104
        my $scheme = $filter_uri->scheme;
105
        if ( ($scheme && $scheme eq "file") || ! $scheme ){
106
            my $path = $filter_uri->path;
107
            #Filters may theoretically be .xsl or .pm files
108
            my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
109
            if ($suffix){
110
                if ( $suffix eq ".xsl"){
111
                    $type = "xslt";
112
                    $filter = $path;
113
                }
114
            }
115
        }
116
    }
117
    return ($type,$filter);
118
}
119
120
sub filter {
121
    my ($self) = @_;
122
    my $filtered = 0;
123
    my $doc = $self->{doc};
124
    my $filter = $self->{filter};
125
    my $filter_type = $self->{filter_type};
126
    if ($doc){
127
        if ($filter && -f $filter){
128
            if ($filter_type){
129
                if ( $filter_type eq 'xslt' ){
130
                    my $xslt = XML::LibXSLT->new();
131
                    my $style_doc = XML::LibXML->load_xml(location => $filter);
132
                    my $stylesheet = $xslt->parse_stylesheet($style_doc);
133
                    if ($stylesheet){
134
                        my $results = $stylesheet->transform($doc);
135
                        my $filtered_record = $stylesheet->output_as_bytes($results);
136
                        if ($filtered_record){
137
                            $self->{filtered_record} = $filtered_record;
138
                            return 1;
139
                        }
140
                    }
141
                }
142
            }
143
        }
144
    }
145
    return 0;
146
}
147
148
sub import_record {
149
    my ($self, $args) = @_;
150
    my $koha_record_numbers = "";
151
    my $errors = [];
152
    my $import_status = "error";
153
    my $match_status = "no_match";
154
155
    my $batch_id = $args->{import_batch_id};
156
    $self->{import_batch_id} = $batch_id;
157
158
    my $matcher = $args->{matcher};
159
    my $framework = $args->{framework};
160
161
    my $metadata_xml = $self->{filtered_record};
162
163
    if ($metadata_xml){
164
        #Convert MARCXML into MARC::Record object
165
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
166
        my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)};
167
        if ($@) {
168
            push(@$errors, { type => 'create_failed', error_msg => "Error converting OAI-PMH filtered metadata into MARC::Record object: $@", record_id => "", }) ;
169
        }
170
171
        if ($self->is_deleted_upstream){
172
173
            #FIXME: Biblio only. Add authority record support later...
174
            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
175
176
            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher, MAX_MATCHES);
177
            if ($number_of_matches > 0){
178
                $match_status = "auto_match"; #See `import_records` table for other options... but this should be the right one.
179
            }
180
            my $results = GetImportRecordMatches($import_record_id); #Only works for biblio...
181
            my $delete_error;
182
183
            my @result_record_numbers = ();
184
            foreach my $result (@$results){
185
                if (my $record_id = $result->{biblionumber}){
186
                    push(@result_record_numbers,$record_id);
187
188
                    #FIXME: Biblio only. Add authority record support later...
189
                    my $error = C4::Biblio::DelBiblio($record_id);
190
                    if ($error){
191
                        $delete_error++;
192
                        push(@$errors, { type => 'delete_failed', error_msg => $error, record_id => $record_id, }) ;
193
                    }
194
                }
195
            }
196
            $koha_record_numbers = join(",",@result_record_numbers);
197
198
            if ($delete_error){
199
                $import_status = "error";
200
                C4::ImportBatch::SetImportBatchStatus($batch_id, 'importing');
201
            } else {
202
                $import_status = "ok";
203
                #Ideally, it would be nice to say what records were deleted via the MARC import functionality, but Koha doesn't have that capacity at the moment, so just clean the batch.
204
                CleanBatch($batch_id);
205
            }
206
207
        } else {
208
            #Import the MARCXML record into Koha
209
210
            #FIXME: Biblio only. Add authority record support later...
211
            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
212
213
            #NOTE: Don't provide item imports since there's no reliable way of tracking changes to them over time.
214
            #my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS');
215
            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher);
216
217
            BatchCommitRecords($batch_id, $framework);
218
219
            my $dbh = C4::Context->dbh();
220
221
            #FIXME: Biblio only. Add authority record support later...
222
            my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?");
223
            $sth->execute($import_record_id);
224
            $koha_record_numbers = $sth->fetchrow_arrayref->[0] || '';
225
226
            $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?");
227
            $sth->execute($import_record_id);
228
            $match_status = $sth->fetchrow_arrayref->[0] || 'no_match';
229
230
            $import_status = "ok";
231
        }
232
    } else {
233
        #There's no filtered metadata...
234
        #Clean the batch, so future imports don't use the same batch.
235
        CleanBatch($batch_id);
236
    }
237
    $self->{status} = $import_status;
238
239
    return ($import_status,$match_status,$koha_record_numbers, $errors);
240
}
241
242
sub save_to_database {
243
    my ($self,$args) = @_;
244
    my $header_identifier = $self->{header_identifier};
245
    my $header_datestamp = $self->{header_datestamp};
246
    my $header_status = $self->{header_status};
247
    my $record = $self->{doc}->toString(1);
248
    my $import_batch_id = $self->{import_batch_id};
249
    my $filter = $self->{filter};
250
    my $status = $self->{status};
251
    my $dbh = C4::Context->dbh;
252
    my $sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, record, import_batch_id, filter, status) VALUES (?, ?, ?, ?, ?, ?, ?)";
253
    my $sth = $dbh->prepare($sql);
254
    $sth->execute($header_identifier,$header_datestamp,$header_status,$record, $import_batch_id, $filter, $status);
255
}
256
257
1;
(-)a/Koha/Schema/Result/ImportOai.pm (+152 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::ImportOai;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::ImportOai
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<import_oai>
19
20
=cut
21
22
__PACKAGE__->table("import_oai");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 header_identifier
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 header_datestamp
40
41
  data_type: 'datetime'
42
  datetime_undef_if_invalid: 1
43
  is_nullable: 0
44
45
=head2 header_status
46
47
  data_type: 'varchar'
48
  is_nullable: 1
49
  size: 45
50
51
=head2 record
52
53
  data_type: 'longtext'
54
  is_nullable: 0
55
56
=head2 upload_timestamp
57
58
  data_type: 'timestamp'
59
  datetime_undef_if_invalid: 1
60
  default_value: current_timestamp
61
  is_nullable: 0
62
63
=head2 status
64
65
  data_type: 'varchar'
66
  is_nullable: 0
67
  size: 45
68
69
=head2 import_batch_id
70
71
  data_type: 'integer'
72
  is_foreign_key: 1
73
  is_nullable: 0
74
75
=head2 filter
76
77
  data_type: 'text'
78
  is_nullable: 0
79
80
=cut
81
82
__PACKAGE__->add_columns(
83
  "import_oai_id",
84
  {
85
    data_type => "integer",
86
    extra => { unsigned => 1 },
87
    is_auto_increment => 1,
88
    is_nullable => 0,
89
  },
90
  "header_identifier",
91
  { data_type => "varchar", is_nullable => 0, size => 45 },
92
  "header_datestamp",
93
  {
94
    data_type => "datetime",
95
    datetime_undef_if_invalid => 1,
96
    is_nullable => 0,
97
  },
98
  "header_status",
99
  { data_type => "varchar", is_nullable => 1, size => 45 },
100
  "record",
101
  { data_type => "longtext", is_nullable => 0 },
102
  "upload_timestamp",
103
  {
104
    data_type => "timestamp",
105
    datetime_undef_if_invalid => 1,
106
    default_value => \"current_timestamp",
107
    is_nullable => 0,
108
  },
109
  "status",
110
  { data_type => "varchar", is_nullable => 0, size => 45 },
111
  "import_batch_id",
112
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
113
  "filter",
114
  { data_type => "text", is_nullable => 0 },
115
);
116
117
=head1 PRIMARY KEY
118
119
=over 4
120
121
=item * L</import_oai_id>
122
123
=back
124
125
=cut
126
127
__PACKAGE__->set_primary_key("import_oai_id");
128
129
=head1 RELATIONS
130
131
=head2 import_batch
132
133
Type: belongs_to
134
135
Related object: L<Koha::Schema::Result::ImportBatch>
136
137
=cut
138
139
__PACKAGE__->belongs_to(
140
  "import_batch",
141
  "Koha::Schema::Result::ImportBatch",
142
  { import_batch_id => "import_batch_id" },
143
  { is_deferrable => 1, on_delete => "RESTRICT", on_update => "RESTRICT" },
144
);
145
146
147
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-04-12 11:02:33
148
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QmCetOjXql0gsAi+wZ74Ng
149
150
151
# You can replace this text with custom code or comments, and it will be preserved on regeneration
152
1;
(-)a/docs/OAIPMH/README (+19 lines)
Line 0 Link Here
1
- Add unit tests
2
- Clean up the code
3
- Add documentation to all code
4
5
6
7
8
FUTURE:
9
- Add support for UNIMARC
10
- Add support for authority records (and holdings/item records)
11
12
- Add default OAI record matching rule?
13
    - I thought about adding a SQL atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex. This needs to be done in Perl.
14
    - Should the rule include other fields like 022, 020, 245 rather than just 001 and 024a?
15
16
- Add entry to Cleanupdatabase.pl cronjob?
17
    - You could remove all import_oai rows older than a certain age?
18
19
- Re-do the paging to use DataTables AJAX? Or, create a centralized/generalized server-side paging function for Koha...
(-)a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql (+15 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS import_oai;
2
CREATE TABLE  import_oai (
3
  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
4
  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
5
  header_datestamp datetime NOT NULL,
6
  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
7
  record longtext CHARACTER SET utf8 NOT NULL,
8
  upload_timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
9
  status varchar(45) CHARACTER SET utf8 NOT NULL,
10
  import_batch_id int(11) NOT NULL,
11
  filter text COLLATE utf8_unicode_ci NOT NULL,
12
  PRIMARY KEY (import_oai_id),
13
  KEY FK_import_oai_1 (import_batch_id),
14
  CONSTRAINT FK_import_oai_1 FOREIGN KEY (import_batch_id) REFERENCES import_batches (import_batch_id)
15
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (+1 lines)
Lines 94-99 Link Here
94
    [% END %]
94
    [% END %]
95
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
95
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
96
	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
96
	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
97
    <li><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></li>
97
    [% END %]
98
    [% END %]
98
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
99
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
99
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
100
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt (+166 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; Manage OAI-PMH record imports
3
[% IF ( import_oai_id ) %]
4
 &rsaquo; Record [% import_oai_id %]
5
[% END %]
6
</title>
7
[% INCLUDE 'doc-head-close.inc' %]
8
<link rel="stylesheet" type="text/css" href="[% interface %]/[% theme %]/css/datatables.css" />
9
[% INCLUDE 'datatables.inc' %]
10
</head>
11
12
<body id="tools_manage-oai-import" class="tools">
13
[% INCLUDE 'header.inc' %]
14
[% INCLUDE 'cat-search.inc' %]
15
16
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a>
17
    [% IF ( import_oai_id ) %]
18
     &rsaquo;
19
     <a href="[% script_name %]">Manage OAI-PMH record imports</a>
20
     &rsaquo; Record [% import_oai_id %]
21
    [% ELSE %]
22
     &rsaquo; Manage OAI-PMH record imports
23
    [% END %]
24
    </div>
25
26
    <div id="doc3" class="yui-t2">
27
        <div id="bd">
28
            <div id="yui-main">
29
                <div class="yui-b">
30
                    [% IF ( import_oai_id ) %]
31
                        [% IF ( view_record ) %]
32
                            <h1>Record [% import_oai_id %]</h1>
33
                            [% IF ( oai_record.record ) %]
34
                                <div style="white-space:pre">[% oai_record.record | xml %]</div>
35
                            [% END %]
36
                        [% ELSIF ( retry ) %]
37
                            <fieldset class="rows">
38
                                <ol>
39
                                    <li>
40
                                        <span class="label">Import status:</span>
41
                                        [% IF ( import_status ) %]
42
                                            [% IF ( import_status == "ok" ) %]
43
                                            OK
44
                                            [% ELSIF ( import_status == "error" ) %]
45
                                            ERROR
46
                                            [% END %]
47
                                        [% END %]
48
                                    </li>
49
                                    [% IF ( errors ) %]
50
                                        [% FOREACH error IN errors %]
51
                                            [% IF ( error ) %]
52
                                                <li>
53
                                                    <span class="label">Error:</span>
54
                                                    [% SWITCH error.type %]
55
                                                    [% CASE 'delete_failed' %]
56
                                                        [% error.error_msg # FIXME: These English messages come straight from C4::Biblio... %]
57
                                                        [% IF ( record_type ) && ( record_type == "biblio" ) %]
58
                                                            <a title="View biblio record" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% error.record_id %]">(View biblio record)</a>
59
                                                        [% END %]
60
                                                    [% CASE 'filter_failed' %]
61
                                                        Filter failed to produce MARCXML.
62
                                                        <a href="/cgi-bin/koha/tools/manage-oai-import.pl?op=view_record&import_oai_id=[% import_oai_id %]">View record [% import_oai_id %]</a>.
63
                                                        You should also review your filter at [% oai_record.filter %].
64
                                                    [% CASE %]
65
                                                        [% error.error_msg %]
66
                                                    [% END %]
67
                                                </li>
68
                                            [% END %]
69
                                        [% END %]
70
                                    [% END %]
71
                                </ol>
72
                            </fieldset>
73
                        [% END %]
74
                    [% ELSE %]
75
                        <h1>Manage OAI-PMH record imports</h1>
76
                        <table>
77
                            <thead>
78
                                <tr>
79
                                    <th>Record identifier</th>
80
                                    <th>Record datestamp</th>
81
                                    <th>Provider status</th>
82
                                    <th>Import status</th>
83
                                    <th>Import batch</th>
84
                                    <th>OAI-PMH record</th>
85
                                    [%# <th>Filter</th> %]
86
                                </tr>
87
                            </thead>
88
                            <tbody>
89
                                [% WHILE (oai_record = oai_records.next) %]
90
                                <tr>
91
                                    <td>[% oai_record.header_identifier %]</td>
92
                                    <td>[% oai_record.header_datestamp %]</td>
93
                                    <td>
94
                                        [% IF ( oai_record.header_status ) %]
95
                                            [% IF ( oai_record.header_status == "deleted" ) %]
96
                                                DELETED
97
                                            [% END %]
98
                                        [% END %]
99
                                    </td>
100
                                    <td>
101
                                        [% IF ( oai_record.status ) %]
102
                                            [% IF ( oai_record.status == "ok" ) %]
103
                                                OK
104
                                            [% ELSIF ( oai_record.status == "error" ) %]
105
                                                ERROR
106
                                                <form action="[% script_name %]">
107
                                                    <input type="hidden" name="op" value="retry" />
108
                                                    <input type="hidden" name="import_oai_id" value="[% oai_record.import_oai_id %]" />
109
                                                    <select name="frameworkcode">
110
                                                        <option value="">Default framework</option>
111
                                                        [% FOREACH framework IN frameworks %]
112
                                                            <option value="[% framework.frameworkcode %]">[% framework.frameworktext %]</option>
113
                                                        [% END %]
114
                                                    </select>
115
                                                    <input type="submit" value="Retry"/>
116
                                                </form>
117
                                                <!-- <a title="Retry import" href="[% script_name %]?op=retry&import_oai_id=[% oai_record.import_oai_id %]">ERROR - Click to retry</a> -->
118
                                            [% END %]
119
120
                                        [% ELSE %]
121
                                            Unknown
122
                                        [% END %]
123
                                    </td>
124
                                    <td>
125
                                        [% IF ( oai_record.import_batch_id ) %]
126
                                            <a title="View import batch" href="/cgi-bin/koha/tools/manage-marc-import.pl?import_batch_id=[% oai_record.import_batch_id %]">View batch [% oai_record.import_batch_id %]</a>
127
                                        [% END %]
128
                                    </td>
129
                                    [%# oai_record.filter %]
130
                                    <td><a title="View OAI-PMH record" href="[% script_name %]?op=view_record&import_oai_id=[% oai_record.import_oai_id %]">View record [% oai_record.import_oai_id %]</a></td>
131
                                </tr>
132
                                [% END %]
133
134
                            </tbody>
135
                        </table>
136
                        <div class="pager">
137
                        [% IF ( page_first ) %]
138
                            <a href="[% script_name %]?page=[% page_first %]">First ([% page_first %])</a>
139
                        [% ELSE %]
140
                            <a class="disabled">First</a>
141
                        [% END %]
142
                        [% IF ( page_previous ) %]
143
                            <a href="[% script_name %]?page=[% page_previous %]">Previous ([% page_previous %])</a>
144
                        [% ELSE %]
145
                            <a class="disabled">Previous</a>
146
                        [% END %]
147
                        [% IF ( page_next ) %]
148
                            <a href="[% script_name %]?page=[% page_next %]">Next ([% page_next %])</a>
149
                        [% ELSE %]
150
                            <a class="disabled">Next</a>
151
                        [% END %]
152
                        [% IF ( page_last ) %]
153
                            <a href="[% script_name %]?page=[% page_last %]">Last ([% page_last %])</a>
154
                        [% ELSE %]
155
                            <a class="disabled">Last</a>
156
                        [% END %]
157
                        </div>
158
                    [% END %]
159
                </div>
160
            </div>
161
            <div class="yui-b">
162
                [% INCLUDE 'tools-menu.inc' %]
163
            </div>
164
        </div>
165
    </div>
166
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+3 lines)
Lines 177-182 Link Here
177
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
177
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
178
    <dt><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC record management</a></dt>
178
    <dt><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC record management</a></dt>
179
    <dd>Managed staged MARC records, including completing and reversing imports</dd>
179
    <dd>Managed staged MARC records, including completing and reversing imports</dd>
180
181
    <dt><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></dt>
182
    <dd>Manage import of OAI-PMH harvested records</dd>
180
    [% END %]
183
    [% END %]
181
184
182
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
185
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl (+78 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:marc="http://www.loc.gov/MARC21/slim"
4
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
5
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
6
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
7
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
8
    <!-- NOTE: This XSLT also adds the OAI-PMH identifier back in as a MARC field -->
9
10
    <!-- Match the root oai:record element -->
11
    <xsl:template match="oai:record">
12
        <!-- Apply templates only when the oai record is for a deleted item -->
13
        <xsl:apply-templates select="oai:header[@status='deleted']" />
14
        <!-- Apply templates only to the child metadata element(s) -->
15
        <xsl:apply-templates select="oai:metadata" />
16
    </xsl:template>
17
18
    <!-- Matches an oai:metadata element -->
19
    <xsl:template match="oai:metadata">
20
        <!-- Only apply further templates to marc:record elements -->
21
        <!-- This prevents the identity transformation from outputting other non-MARC metadata formats -->
22
        <xsl:apply-templates select="//marc:record"/>
23
    </xsl:template>
24
25
    <!-- We need to create a MARCXML record from OAI records marked "deleted" to handle OAI deletions correctly in Koha -->
26
    <xsl:template match="oai:header[@status='deleted']">
27
        <xsl:element name="record" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
28
            xmlns="http://www.loc.gov/MARC21/slim">
29
            <xsl:attribute name="xsi:schemaLocation">http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</xsl:attribute>
30
            <xsl:call-template name="add_oai"/>
31
        </xsl:element>
32
    </xsl:template>
33
34
    <!-- Identity transformation: this template copies attributes and nodes -->
35
    <xsl:template match="@* | node()">
36
        <!-- Create a copy of this attribute or node -->
37
        <xsl:copy>
38
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
39
            <xsl:apply-templates select="@* | node()" />
40
        </xsl:copy>
41
    </xsl:template>
42
43
44
    <xsl:template match="marc:record">
45
        <xsl:copy>
46
            <!-- Apply all relevant templates for all attributes and elements -->
47
            <xsl:apply-templates select="@* | node()"/>
48
49
            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
50
            <xsl:call-template name="add_oai"/>
51
52
            <!-- Newline -->
53
            <xsl:text>&#xa;</xsl:text>
54
        </xsl:copy>
55
    </xsl:template>
56
57
    <!-- Template for adding the OAI-PMH identifier as 024$a -->
58
    <xsl:template name="add_oai">
59
        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
60
            <xsl:attribute name="ind1"><xsl:text>7</xsl:text></xsl:attribute>
61
            <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
62
            <xsl:attribute name="tag">024</xsl:attribute>
63
            <xsl:element name="subfield">
64
                <xsl:attribute name="code">a</xsl:attribute>
65
                <xsl:value-of select="/oai:record/oai:header/oai:identifier"/>
66
            </xsl:element>
67
            <xsl:element name="subfield">
68
                <xsl:attribute name="code">2</xsl:attribute>
69
                <xsl:text>uri</xsl:text>
70
            </xsl:element>
71
         </xsl:element>
72
    </xsl:template>
73
74
    <!-- Delete items -->
75
    <xsl:template match="marc:datafield[@tag='952']">
76
    </xsl:template>
77
78
</xsl:stylesheet>
(-)a/svc/import_oai (+124 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use CGI qw ( -utf8 );
23
use XML::Simple;
24
25
use C4::Auth qw/check_api_auth/;
26
use C4::Context;
27
use C4::ImportBatch;
28
use C4::Matcher;
29
use C4::Biblio;
30
31
use Koha::OAI::Client::Record;
32
33
my $query = new CGI;
34
binmode STDOUT, ':encoding(UTF-8)';
35
36
my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
37
unless ($status eq "ok") {
38
    print $query->header(-type => 'text/xml', -status => '403 Forbidden');
39
    print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1);
40
    exit 0;
41
}
42
43
my $xml;
44
if ($query->request_method eq "POST") {
45
    $xml = $query->param('xml');
46
}
47
if ($xml) {
48
    my %params = $query->Vars;
49
    my $result = import_oai($xml, \%params );
50
    print $query->header(-type => 'text/xml');
51
    print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1);
52
} else {
53
    print $query->header(-type => 'text/xml', -status => '400 Bad Request');
54
}
55
56
exit 0;
57
58
sub import_oai {
59
    my ($inxml, $params) = @_;
60
61
    my $result = {};
62
    my $result_errors = [];
63
64
    my $filter      = delete $params->{filter}      || '';
65
    my $framework   = delete $params->{framework}   || '';
66
    my $record_type   = delete $params->{record_type}   || 'biblio';
67
68
    if (my $matcher_code = delete $params->{match}) {
69
        $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code);
70
    }
71
72
    #Get matching rule matcher
73
    my $matcher = C4::Matcher->new($record_type);
74
    $matcher = C4::Matcher->fetch($params->{matcher_id});
75
76
    #Create record object
77
    my $oai_record = Koha::OAI::Client::Record->new({
78
        xml_string => $inxml,
79
    });
80
81
    unless ($params->{comments}){
82
        $params->{comments} = "OAI-PMH import";
83
        if ($oai_record->{header_identifier}){
84
            $params->{comments} .= ": $oai_record->{header_identifier}";
85
        }
86
    }
87
    my $batch_id = GetWebserviceBatchId($params);
88
    unless ($batch_id) {
89
        $result->{'status'} = "failed";
90
        $result->{'error'} = "Batch create error";
91
        return $result;
92
    }
93
94
    #Filter OAI-PMH into MARCXML
95
    $oai_record->set_filter($filter);
96
    my $filtered = $oai_record->filter();
97
    if (!$filtered){
98
        push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
99
    }
100
101
    my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
102
        matcher => $matcher,
103
        import_batch_id => $batch_id,
104
        framework => $framework,
105
    });
106
    if (@$errors){
107
        push(@$result_errors,@$errors);
108
    }
109
110
    $oai_record->save_to_database();
111
112
    $result->{'match_status'} = $match_status;
113
    $result->{'import_batch_id'} = $batch_id;
114
    $result->{'koha_record_numbers'} = $koha_record_numbers;
115
116
    if ($import_status && $import_status eq "ok"){
117
        $result->{'status'} = "ok";
118
    } else {
119
        $result->{'status'} = "failed";
120
        $result->{'errors'} = {error => $result_errors};
121
    }
122
123
    return $result;
124
}
(-)a/t/Import/bib-deleted.xml (+7 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns="http://www.openarchives.org/OAI/2.0/">
3
  <header status="deleted">
4
    <identifier>oai:koha-community.org:5000</identifier>
5
    <datestamp>2015-12-22T18:46:29Z</datestamp>
6
  </header>
7
</record>
(-)a/t/Import/bib-oaidc.xml (+19 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns="http://www.openarchives.org/OAI/2.0/">
3
  <header>
4
    <identifier>oai:koha-community.org:5000</identifier>
5
    <datestamp>2015-12-21T18:46:29Z</datestamp>
6
  </header>
7
  <metadata>
8
    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
9
        <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
10
          <dc:title>Everything you never wanted to know about OAI-PMH : a primer /</dc:title>
11
          <dc:creator>
12
            Cook, David
13
          </dc:creator>
14
          <dc:type>text</dc:type>
15
          <dc:language>eng</dc:language>
16
        </oai_dc:dc>
17
    </metadata>
18
  </metadata>
19
</record>
(-)a/t/Import/bib.xml (+28 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns="http://www.openarchives.org/OAI/2.0/">
3
  <header>
4
    <identifier>oai:koha-community.org:5000</identifier>
5
    <datestamp>2015-12-21T18:46:29Z</datestamp>
6
  </header>
7
  <metadata>
8
    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
9
      <record xmlns="http://www.loc.gov/MARC21/slim" type="Bibliographic">
10
        <leader>01005cam a22003377a 4500</leader>
11
        <controlfield tag="001">123456789</controlfield>
12
        <controlfield tag="005">20151221185246.0</controlfield>
13
        <controlfield tag="008">010203s2004    nyu           000 0 eng u</controlfield>
14
        <datafield ind1="1" tag="100" ind2=" ">
15
          <subfield code="a">Cook, David</subfield>
16
        </datafield>
17
        <datafield ind1="1" ind2="0" tag="245">
18
          <subfield code="a">Everything you never wanted to know about OAI-PMH :</subfield>
19
          <subfield code="b">a primer /</subfield>
20
          <subfield code="c">David Cook</subfield>
21
        </datafield>
22
        <datafield ind1=" " tag="911" ind2=" ">
23
          <subfield code="a">Test</subfield>
24
        </datafield>
25
      </record>
26
    </metadata>
27
  </metadata>
28
</record>
(-)a/tools/manage-oai-import.pl (-1 / +175 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use POSIX qw//;
22
23
use Koha::Database;
24
use Koha::BiblioFrameworks;
25
use Koha::OAI::Client::Record;
26
27
use C4::Auth;
28
use C4::Output;
29
use C4::Koha;
30
use C4::Context;
31
use C4::Matcher;
32
use C4::ImportBatch;
33
34
my $input = new CGI;
35
my $op = $input->param('op') || 'list';
36
my $page_number = $input->param('page') && $input->param('page') =~ /^\d+$/ ? $input->param('page') : 1; #Only permit numeric parameter
37
my $import_oai_id = $input->param('import_oai_id');
38
my $frameworkcode = $input->param('frameworkcode') // '';
39
40
my $schema = Koha::Database->new()->schema();
41
my $resultset = $schema->resultset('ImportOai');
42
43
my ($template, $loggedinuser, $cookie) =
44
    get_template_and_user({template_name => "tools/manage-oai-import.tt",
45
        query => $input,
46
        type => "intranet",
47
        authnotrequired => 0,
48
        flagsrequired => {tools => 'manage_staged_marc'},
49
    });
50
51
$template->{VARS}->{ script_name } = "/cgi-bin/koha/tools/manage-oai-import.pl";
52
53
if ($import_oai_id){
54
    $template->{VARS}->{ import_oai_id } = $import_oai_id;
55
    my $import_oai_record = $resultset->find($import_oai_id);
56
    $template->{VARS}->{ oai_record } = $import_oai_record;
57
58
    if ($op eq "view_record"){
59
        $template->{VARS}->{ view_record } = 1;
60
    } elsif ($op eq "retry"){
61
        if ($import_oai_record){
62
            my $result_errors = [];
63
64
            my $oai_record = Koha::OAI::Client::Record->new({
65
                xml_string => $import_oai_record->record,
66
            });
67
            if ($oai_record){
68
                #Filter OAI-PMH into MARCXML
69
                my $filter = $import_oai_record->filter;
70
                $oai_record->set_filter($filter);
71
                my $filtered = $oai_record->filter();
72
                if (!$filtered){
73
                    push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
74
                }
75
76
                my $import_batch_id = $import_oai_record->import_batch_id;
77
                if ($import_batch_id){
78
                    #Reset the batch before re-trying the import
79
                    C4::ImportBatch::CleanBatch($import_batch_id);
80
81
                    my $import_batch_rs = $schema->resultset('ImportBatch');
82
                    my $import_batch = $import_batch_rs->find($import_batch_id);
83
84
                    my $matcher_id = $import_batch->matcher_id;
85
                    my $matcher = C4::Matcher->fetch($matcher_id);
86
87
                    my $record_type = $import_batch->record_type;
88
                    $template->{VARS}->{ record_type } = $record_type;
89
90
                    my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
91
                        matcher => $matcher,
92
                        import_batch_id => $import_batch_id,
93
                        framework => $frameworkcode,
94
                    });
95
                    if (@$errors){
96
                        push(@$result_errors,@$errors);
97
                    }
98
99
                    if ($import_status){
100
                        if ($import_status eq 'ok'){
101
                            $import_oai_record->status("ok");
102
                            $import_oai_record->update();
103
                        } else {
104
                            $template->{VARS}->{ retry } = 1;
105
                            $template->{VARS}->{ import_status } = $import_status;
106
                        }
107
                    }
108
                }
109
            }
110
            $template->{VARS}->{ errors } = $result_errors;
111
        }
112
    }
113
}
114
115
if ($op && $op eq "list"){
116
    my @frameworks = Koha::BiblioFrameworks->as_list();
117
    $template->{VARS}->{ frameworks } = \@frameworks;
118
119
    #NOTE: It would be preferable if we centralized server-side paging code with generic functions...
120
    #START PAGING
121
    $template->{VARS}->{ page_number } = $page_number;
122
123
    #Get grand total in the database
124
    my $total_rows = $resultset->count;
125
126
    my $number_of_rows = 10;
127
    my $number_of_pages = POSIX::ceil($total_rows / $number_of_rows);
128
129
    if ($page_number > 1){
130
        $template->{VARS}->{ page_first } = 1;
131
    }
132
    if ($number_of_pages > 1 && $page_number != $number_of_pages){
133
        $template->{VARS}->{ page_last } = $number_of_pages;
134
    }
135
136
    #Do the search and define a limit
137
    my $results = $resultset->search(
138
        undef,
139
        {
140
            rows => $number_of_rows,
141
            order_by => { -desc => 'upload_timestamp' },
142
        },
143
    );
144
145
    my $page = $results->page($page_number);
146
    $template->{VARS}->{ oai_records } = $page;
147
148
    my $current_page_rows = $page->count();
149
150
    if ($current_page_rows){
151
        if ($page_number == 1) {
152
            #Can't page previous to the first page
153
            if ( $current_page_rows == $total_rows ){
154
                #Can't page past the total count
155
            } else {
156
                #Signal that there's another page
157
                $template->{VARS}->{ page_next } = $page_number + 1;
158
            }
159
        } else {
160
            #Signal that there's a previous page
161
            $template->{VARS}->{ page_previous } = $page_number - 1;
162
163
            #Total rows of all previous pages
164
            my $previous_pages_rows = ( $page_number - 1 ) * $number_of_rows;
165
166
            #If current and past rows are less than total rows, there must still be more to show
167
            if ( ($current_page_rows + $previous_pages_rows) < $total_rows){
168
                $template->{VARS}->{ page_next } = $page_number + 1;
169
            }
170
        }
171
    }
172
    #END PAGING
173
}
174
175
output_html_with_http_headers $input, $cookie, $template->output;

Return to bug 10662