View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/Koha/OAI/Client/Record.pm (+249 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Record;
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use XML::LibXSLT;
24
use MARC::Record;
25
26
use C4::Context;
27
use C4::Biblio;
28
use C4::ImportBatch;
29
use C4::Matcher;
30
31
use constant MAX_MATCHES => 99999; #NOTE: This is an arbitrary value. We want to get all matches.
32
33
sub new {
34
    my ($class, $args) = @_;
35
    $args = {} unless defined $args;
36
37
    if (my $inxml = $args->{xml_string}){
38
39
        #Parse the XML string into a XML::LibXML object
40
        my $doc = XML::LibXML->load_xml(string => $inxml, { no_blanks => 1 });
41
        $args->{doc} = $doc;
42
        #NOTE: Don't load blank nodes...
43
44
        #Get the root element
45
        my $root = $doc->documentElement;
46
47
        #Register namespaces for searching purposes
48
        my $xpc = XML::LibXML::XPathContext->new();
49
        $xpc->registerNs('oai','http://www.openarchives.org/OAI/2.0/');
50
51
        my $xpath_identifier = XML::LibXML::XPathExpression->new("oai:header/oai:identifier");
52
        my $identifier = $xpc->findnodes($xpath_identifier,$root)->shift;
53
        #my $identifier_string = $identifier->textContent;
54
        $args->{header_identifier} = $identifier->textContent;
55
56
        my $xpath_datestamp = XML::LibXML::XPathExpression->new("oai:header/oai:datestamp");
57
        my $datestamp = $xpc->findnodes($xpath_datestamp,$root)->shift;
58
        #my $datestamp_string = $datestamp->textContent;
59
        $args->{header_datestamp} = $datestamp->textContent;
60
61
        my $xpath_status = XML::LibXML::XPathExpression->new(q{oai:header/@status});
62
        my $status_node = $xpc->findnodes($xpath_status,$root)->shift;
63
        #my $status_string = $status_node ? $status_node->textContent : "";
64
        $args->{header_status} = $status_node ? $status_node->textContent : "";
65
    }
66
67
    return bless ($args, $class);
68
}
69
70
sub is_deleted_upstream {
71
    my ($self, $args) = @_;
72
    if ($self->{header_status}){
73
        if ($self->{header_status} eq "deleted"){
74
            return 1;
75
        }
76
    }
77
    return 0;
78
}
79
80
sub filter {
81
    my ($self, $args) = @_;
82
    my $doc = $self->{doc};
83
    my $filter = $args->{filter};
84
    $self->{filter} = $filter; #FIXME
85
    #FIXME: Check that it's an XSLT here...
86
    if ( -f $filter ){
87
        #Filter is a valid filepath
88
89
        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here... (especially for persistent environments...)
90
        my $xslt = XML::LibXSLT->new();
91
        my $style_doc = XML::LibXML->load_xml(location => $filter);
92
        my $stylesheet = $xslt->parse_stylesheet($style_doc);
93
        if ($stylesheet){
94
            my $results = $stylesheet->transform($doc);
95
            my $metadata_xml = $stylesheet->output_as_bytes($results);
96
            #If the XSLT outputs nothing, then we don't meet the following condition, and we'll return 0 instead.
97
            if ($metadata_xml){
98
                $self->{filtered_record} = $metadata_xml;
99
                return 1;
100
            }
101
        }
102
    }
103
    return 0;
104
}
105
106
107
108
109
110
111
sub import_record {
112
    my ($self, $args) = @_;
113
    my $koha_record_numbers = "";
114
    my $errors = [];
115
    my $import_status = "error";
116
    my $match_status = "no_match";
117
118
    my $batch_id = $args->{import_batch_id};
119
    $self->{import_batch_id} = $batch_id; #FIXME
120
    my $matcher = $args->{matcher};
121
    my $framework = $args->{framework};
122
    my $import_mode = $args->{import_mode};
123
124
    my $metadata_xml = $self->{filtered_record};
125
126
    if ($metadata_xml){
127
        #Convert MARCXML into MARC::Record object
128
        my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21';
129
        my $marc_record = eval {MARC::Record::new_from_xml( $metadata_xml, "utf8", $marcflavour)};
130
        if ($@) {
131
            warn "Error converting OAI-PMH filtered metadata into MARC::Record object: $@";
132
            #FIXME: Improve error handling
133
        }
134
135
        if ($self->is_deleted_upstream){
136
137
=pod
138
            my @matches = $matcher->get_matches($marc_record, MAX_MATCHES);
139
            if (@matches){
140
                $match_status = "matched";
141
            }
142
            my $delete_error;
143
            foreach my $match (@matches){
144
                    if (my $record_id = $match->{record_id}){
145
                        #FIXME: This is biblio specific... what about authority records?
146
                        my $error = C4::Biblio::DelBiblio($record_id);
147
                        if ($error){
148
                            $delete_error++;
149
                            $koha_record_numbers = [];
150
                            push(@$koha_record_numbers,$record_id);
151
152
                            #FIXME: Find a better way of sending the errors in a predictable way...
153
                            push(@$errors,{"record_id" => $record_id, "error" => $error, });
154
                        }
155
                    }
156
157
            }
158
159
            #If there are no delete errors, then the import was ok
160
            if ( ! $delete_error){
161
                $import_status = "ok";
162
            }
163
            #Deleted records will never actually have an records in them, so always mark them as cleaned so that other imports don't try to pick up the same batch.
164
            C4::ImportBatch::SetImportBatchStatus($batch_id, 'cleaned');
165
=cut
166
            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
167
            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher, MAX_MATCHES);
168
            if ($number_of_matches > 0){
169
                $match_status = "auto_match"; #See `import_records` table for other options... but this should be the right one.
170
            }
171
            my $results = GetImportRecordMatches($import_record_id); #Only works for biblio...
172
            my $delete_error;
173
174
            my @result_record_numbers = ();
175
            foreach my $result (@$results){
176
                if (my $record_id = $result->{biblionumber}){
177
                    push(@result_record_numbers,$record_id);
178
179
                    #FIXME: This is biblio specific... what about authority records?
180
                    my $error = C4::Biblio::DelBiblio($record_id);
181
                    if ($error){
182
                        $delete_error++;
183
                        push(@$errors, { type => 'delete_failed', error_msg => $error, record_id => $record_id, }) ;
184
                    }
185
                }
186
            }
187
            $koha_record_numbers = join(",",@result_record_numbers); #FIXME: check that this works...
188
189
            if ($delete_error){
190
                $import_status = "error";
191
                C4::ImportBatch::SetImportBatchStatus($batch_id, 'importing');
192
            } else {
193
                $import_status = "ok";
194
                #Ideally, it would be nice to say what records were deleted, but Koha doesn't have that capacity at the moment, so just clean the batch.
195
                CleanBatch($batch_id);
196
            }
197
198
199
200
201
        } else {
202
            #Import the MARCXML record into Koha
203
            my $import_record_id = AddBiblioToBatch($batch_id, 0, $marc_record, "utf8", int(rand(99999)));
204
            #FIXME: Don't allow item imports do to the nature of OAI-PMH records updating over time...
205
            #my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 'UPDATE COUNTS');
206
            my $number_of_matches =  BatchFindDuplicates($batch_id, $matcher);
207
208
            # XXX we are ignoring the result of this;
209
            BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct';
210
211
            my $dbh = C4::Context->dbh();
212
            my $sth = $dbh->prepare("SELECT matched_biblionumber FROM import_biblios WHERE import_record_id =?");
213
            $sth->execute($import_record_id);
214
            $koha_record_numbers = $sth->fetchrow_arrayref->[0] || '';
215
            $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id =?");
216
            $sth->execute($import_record_id);
217
218
            $match_status = $sth->fetchrow_arrayref->[0] || 'no_match';
219
            $import_status = "ok";
220
        }
221
    } else {
222
        #There's no filtered metadata...
223
        #Clean the batch, so future imports don't use the same batch.
224
        CleanBatch($batch_id);
225
    }
226
    $self->{status} = $import_status; #FIXME
227
    #$self->save_to_database();
228
    return ($import_status,$match_status,$koha_record_numbers, $errors);
229
}
230
231
sub save_to_database {
232
    my ($self,$args) = @_;
233
234
    my $header_identifier = $self->{header_identifier};
235
    my $header_datestamp = $self->{header_datestamp};
236
    my $header_status = $self->{header_status};
237
    my $metadata = $self->{doc}->toString(1);
238
    my $import_batch_id = $self->{import_batch_id};
239
    my $filter = $self->{filter};
240
    my $status = $self->{status};
241
242
    my $dbh = C4::Context->dbh;
243
    my $sql = "INSERT INTO import_oai (header_identifier, header_datestamp, header_status, metadata, import_batch_id, filter, status) VALUES (?, ?, ?, ?, ?, ?, ?)";
244
    my $sth = $dbh->prepare($sql);
245
    $sth->execute($header_identifier,$header_datestamp,$header_status,$metadata, $import_batch_id, $filter, $status);
246
}
247
248
249
1;
(-)a/Koha/Schema/Result/ImportOai.pm (+152 lines)
Line 0 Link Here
1
use utf8;
2
package Koha::Schema::Result::ImportOai;
3
4
# Created by DBIx::Class::Schema::Loader
5
# DO NOT MODIFY THE FIRST PART OF THIS FILE
6
7
=head1 NAME
8
9
Koha::Schema::Result::ImportOai
10
11
=cut
12
13
use strict;
14
use warnings;
15
16
use base 'DBIx::Class::Core';
17
18
=head1 TABLE: C<import_oai>
19
20
=cut
21
22
__PACKAGE__->table("import_oai");
23
24
=head1 ACCESSORS
25
26
=head2 import_oai_id
27
28
  data_type: 'integer'
29
  extra: {unsigned => 1}
30
  is_auto_increment: 1
31
  is_nullable: 0
32
33
=head2 header_identifier
34
35
  data_type: 'varchar'
36
  is_nullable: 0
37
  size: 45
38
39
=head2 header_datestamp
40
41
  data_type: 'datetime'
42
  datetime_undef_if_invalid: 1
43
  is_nullable: 0
44
45
=head2 header_status
46
47
  data_type: 'varchar'
48
  is_nullable: 1
49
  size: 45
50
51
=head2 metadata
52
53
  data_type: 'longtext'
54
  is_nullable: 0
55
56
=head2 last_modified
57
58
  data_type: 'timestamp'
59
  datetime_undef_if_invalid: 1
60
  default_value: current_timestamp
61
  is_nullable: 0
62
63
=head2 status
64
65
  data_type: 'varchar'
66
  is_nullable: 0
67
  size: 45
68
69
=head2 import_batch_id
70
71
  data_type: 'integer'
72
  is_foreign_key: 1
73
  is_nullable: 0
74
75
=head2 filter
76
77
  data_type: 'text'
78
  is_nullable: 0
79
80
=cut
81
82
__PACKAGE__->add_columns(
83
  "import_oai_id",
84
  {
85
    data_type => "integer",
86
    extra => { unsigned => 1 },
87
    is_auto_increment => 1,
88
    is_nullable => 0,
89
  },
90
  "header_identifier",
91
  { data_type => "varchar", is_nullable => 0, size => 45 },
92
  "header_datestamp",
93
  {
94
    data_type => "datetime",
95
    datetime_undef_if_invalid => 1,
96
    is_nullable => 0,
97
  },
98
  "header_status",
99
  { data_type => "varchar", is_nullable => 1, size => 45 },
100
  "metadata",
101
  { data_type => "longtext", is_nullable => 0 },
102
  "last_modified",
103
  {
104
    data_type => "timestamp",
105
    datetime_undef_if_invalid => 1,
106
    default_value => \"current_timestamp",
107
    is_nullable => 0,
108
  },
109
  "status",
110
  { data_type => "varchar", is_nullable => 0, size => 45 },
111
  "import_batch_id",
112
  { data_type => "integer", is_foreign_key => 1, is_nullable => 0 },
113
  "filter",
114
  { data_type => "text", is_nullable => 0 },
115
);
116
117
=head1 PRIMARY KEY
118
119
=over 4
120
121
=item * L</import_oai_id>
122
123
=back
124
125
=cut
126
127
__PACKAGE__->set_primary_key("import_oai_id");
128
129
=head1 RELATIONS
130
131
=head2 import_batch
132
133
Type: belongs_to
134
135
Related object: L<Koha::Schema::Result::ImportBatch>
136
137
=cut
138
139
__PACKAGE__->belongs_to(
140
  "import_batch",
141
  "Koha::Schema::Result::ImportBatch",
142
  { import_batch_id => "import_batch_id" },
143
  { is_deferrable => 1, on_delete => "RESTRICT", on_update => "RESTRICT" },
144
);
145
146
147
# Created by DBIx::Class::Schema::Loader v0.07042 @ 2016-04-12 11:02:33
148
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:QmCetOjXql0gsAi+wZ74Ng
149
150
151
# You can replace this text with custom code or comments, and it will be preserved on regeneration
152
1;
(-)a/docs/OAIPMH/README (+20 lines)
Line 0 Link Here
1
TODO:
2
- Change `import_oai` database table?
3
    - import_oai's "metadata" should actually be "oai_record"... so it's not so confusing... it's NOT the metadata element... but rather the whole OAI record.
4
    - Rename "last_modified" to "upload_timestamp"
5
- Fix https://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=15541
6
7
- Add documentation to all code...
8
- Add unit tests
9
- Clean up the code
10
- Resolve all TODO/FIXME comments in the code
11
12
    
13
FUTURE:
14
- Add support for authority records and possibly holdings records?
15
- Add default OAI record matching rule?
16
    - I thought about adding a SQL atomic update 'bug_10662-Add_oai_record_matching_rule.sql', but adding matching rules seems complex. This needs to be done in Perl.
17
    - Should the rule include other fields like 022, 020, 245 rather than just 001 and 024a?
18
- Add entry to Cleanupdatabase.pl cronjob?
19
    - You could remove all import_oai rows older than a certain age?
20
- Re-do the paging to use DataTables AJAX? Or, create a centralized/generalized server-side paging function for Koha...
(-)a/installer/data/mysql/atomicupdate/bug_10662-Build_import_oai_table.sql (+15 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS import_oai;
2
CREATE TABLE  import_oai (
3
  import_oai_id int(10) unsigned NOT NULL AUTO_INCREMENT,
4
  header_identifier varchar(45) CHARACTER SET utf8 NOT NULL,
5
  header_datestamp datetime NOT NULL,
6
  header_status varchar(45) CHARACTER SET utf8 DEFAULT NULL,
7
  metadata longtext CHARACTER SET utf8 NOT NULL,
8
  last_modified timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
9
  status varchar(45) CHARACTER SET utf8 NOT NULL,
10
  import_batch_id int(11) NOT NULL,
11
  filter text COLLATE utf8_unicode_ci NOT NULL,
12
  PRIMARY KEY (import_oai_id),
13
  KEY FK_import_oai_1 (import_batch_id),
14
  CONSTRAINT FK_import_oai_1 FOREIGN KEY (import_batch_id) REFERENCES import_batches (import_batch_id)
15
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/tools-menu.inc (+1 lines)
Lines 92-97 Link Here
92
    [% END %]
92
    [% END %]
93
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
93
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
94
	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
94
	<li><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC management</a></li>
95
    <li><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></li>
95
    [% END %]
96
    [% END %]
96
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
97
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
97
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
98
    <li><a href="/cgi-bin/koha/tools/upload-cover-image.pl">Upload local cover image</a></li>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/manage-oai-import.tt (+154 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Tools &rsaquo; Manage OAI-PMH record imports
3
[% IF ( import_oai_id ) %]
4
 &rsaquo; Record [% import_oai_id %]
5
[% END %]
6
</title>
7
[% INCLUDE 'doc-head-close.inc' %]
8
<link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
9
[% INCLUDE 'datatables.inc' %]
10
</head>
11
12
<body id="tools_manage-oai-import" class="tools">
13
[% INCLUDE 'header.inc' %]
14
[% INCLUDE 'cat-search.inc' %]
15
16
    <div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/tools/tools-home.pl">Tools</a>
17
    [% IF ( import_oai_id ) %]
18
     &rsaquo;
19
     <a href="[% script_name %]">Manage OAI-PMH record imports</a>
20
     &rsaquo; Record [% import_oai_id %]
21
    [% ELSE %]
22
     &rsaquo; Manage OAI-PMH record imports
23
    [% END %]
24
    </div>
25
26
    <div id="doc3" class="yui-t2">
27
        <div id="bd">
28
            <div id="yui-main">
29
                <div class="yui-b">
30
                    [% IF ( import_oai_id ) %]
31
                        [% IF ( view_record ) %]
32
                            <h1>Record [% import_oai_id %]</h1>
33
                            [% IF ( oai_record.metadata ) %]
34
                                <div style="white-space:pre">[% oai_record.metadata | xml %]</div>
35
                            [% END %]
36
                        [% ELSIF ( retry ) %]
37
                            <fieldset class="rows">
38
                                <ol>
39
                                    <li>
40
                                        <span class="label">Import status:</span>
41
                                        [% IF ( import_status ) %]
42
                                            [% IF ( import_status == "ok" ) %]
43
                                            OK
44
                                            [% ELSIF ( import_status == "error" ) %]
45
                                            ERROR
46
                                            [% END %]
47
                                        [% END %]
48
                                    </li>
49
                                    [% IF ( errors ) %]
50
                                        [% FOREACH error IN errors %]
51
                                            [% IF ( error ) %]
52
                                                <li>
53
                                                    <span class="label">Error:</span>
54
                                                    [% SWITCH error.type %]
55
                                                    [% CASE 'delete_failed' %]
56
                                                        [% error.error_msg # FIXME: These English messages come straight from C4::Biblio... %]
57
                                                        [% IF ( record_type ) && ( record_type == "biblio" ) %]
58
                                                            <a title="View biblio record" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% error.record_id %]">(View biblio record)</a>
59
                                                        [% END %]
60
                                                    [% CASE 'filter_failed' %]
61
                                                        Filter failed to produce MARCXML.
62
                                                        Review the metadata at <a href="/cgi-bin/koha/tools/manage-oai-import.pl?op=view_record&import_oai_id=[% import_oai_id %]">View record [% import_oai_id %]</a>.
63
                                                        You should also review your filter at [% oai_record.filter %].
64
                                                    [% CASE %]
65
                                                        [% error.error_msg %]
66
                                                    [% END %]
67
                                                </li>
68
                                            [% END %]
69
                                        [% END %]
70
                                    [% END %]
71
                                </ol>
72
                            </fieldset>
73
                        [% END %]
74
                    [% ELSE %]
75
                        <h1>Manage OAI-PMH record imports</h1>
76
                        <table>
77
                            <thead>
78
                                <tr>
79
                                    <th>Record identifier</th>
80
                                    <th>Record datestamp</th>
81
                                    <th>Provider status</th>
82
                                    <th>Import status</th>
83
                                    <th>Import batch</th>
84
                                    <th>OAI-PMH record</th>
85
                                    [%# <th>Filter</th> %]
86
                                </tr>
87
                            </thead>
88
                            <tbody>
89
                                [% WHILE (oai_record = oai_records.next) %]
90
                                <tr>
91
                                    <td>[% oai_record.header_identifier %]</td>
92
                                    <td>[% oai_record.header_datestamp %]</td>
93
                                    <td>
94
                                        [% IF ( oai_record.header_status ) %]
95
                                            [% IF ( oai_record.header_status == "deleted" ) %]
96
                                                DELETED
97
                                            [% END %]
98
                                        [% END %]
99
                                    </td>
100
                                    <td>
101
                                        [% IF ( oai_record.status ) %]
102
                                            [% IF ( oai_record.status == "ok" ) %]
103
                                                OK
104
                                            [% ELSIF ( oai_record.status == "error" ) %]
105
                                                <a title="Retry import" href="[% script_name %]?op=retry&import_oai_id=[% oai_record.import_oai_id %]">ERROR - Click to retry</a>
106
                                            [% END %]
107
108
                                        [% ELSE %]
109
                                            Unknown
110
                                        [% END %]
111
                                    </td>
112
                                    <td>
113
                                        [% IF ( oai_record.import_batch_id ) %]
114
                                            <a title="View import batch" href="/cgi-bin/koha/tools/manage-marc-import.pl?import_batch_id=[% oai_record.import_batch_id %]">View batch [% oai_record.import_batch_id %]</a>
115
                                        [% END %]
116
                                    </td>
117
                                    [%# oai_record.filter %]
118
                                    <td><a title="View OAI-PMH record" href="[% script_name %]?op=view_record&import_oai_id=[% oai_record.import_oai_id %]">View record [% oai_record.import_oai_id %]</a></td>
119
                                </tr>
120
                                [% END %]
121
122
                            </tbody>
123
                        </table>
124
                        <div class="pager">
125
                        [% IF ( page_first ) %]
126
                            <a href="[% script_name %]?page=[% page_first %]">First ([% page_first %])</a>
127
                        [% ELSE %]
128
                            <a class="disabled">First</a>
129
                        [% END %]
130
                        [% IF ( page_previous ) %]
131
                            <a href="[% script_name %]?page=[% page_previous %]">Previous ([% page_previous %])</a>
132
                        [% ELSE %]
133
                            <a class="disabled">Previous</a>
134
                        [% END %]
135
                        [% IF ( page_next ) %]
136
                            <a href="[% script_name %]?page=[% page_next %]">Next ([% page_next %])</a>
137
                        [% ELSE %]
138
                            <a class="disabled">Next</a>
139
                        [% END %]
140
                        [% IF ( page_last ) %]
141
                            <a href="[% script_name %]?page=[% page_last %]">Last ([% page_last %])</a>
142
                        [% ELSE %]
143
                            <a class="disabled">Last</a>
144
                        [% END %]
145
                        </div>
146
                    [% END %]
147
                </div>
148
            </div>
149
            <div class="yui-b">
150
                [% INCLUDE 'tools-menu.inc' %]
151
            </div>
152
        </div>
153
    </div>
154
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/tools-home.tt (+3 lines)
Lines 177-182 Link Here
177
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
177
    [% IF ( CAN_user_tools_manage_staged_marc ) %]
178
    <dt><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC record management</a></dt>
178
    <dt><a href="/cgi-bin/koha/tools/manage-marc-import.pl">Staged MARC record management</a></dt>
179
    <dd>Managed staged MARC records, including completing and reversing imports</dd>
179
    <dd>Managed staged MARC records, including completing and reversing imports</dd>
180
181
    <dt><a href="/cgi-bin/koha/tools/manage-oai-import.pl">OAI-PMH import management</a></dt>
182
    <dd>Manage import of OAI-PMH harvested records</dd>
180
    [% END %]
183
    [% END %]
181
184
182
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
185
    [% IF ( CAN_user_tools_upload_local_cover_images ) %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/OAI2MARC21slim.xsl (+74 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:marc="http://www.loc.gov/MARC21/slim"
4
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
5
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
6
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
7
    <!-- NOTE: This XSLT strips the OAI-PMH wrapper from the metadata. -->
8
    <!-- NOTE: This XSLT also adds the OAI-PMH identifier back in as a MARC field -->
9
10
    <!-- Match the root oai:record element -->
11
    <xsl:template match="oai:record">
12
        <!-- Apply templates only when the oai record is for a deleted item -->
13
        <xsl:apply-templates select="oai:header[@status='deleted']" />
14
        <!-- Apply templates only to the child metadata element(s) -->
15
        <xsl:apply-templates select="oai:metadata" />
16
    </xsl:template>
17
18
    <!-- Matches an oai:metadata element -->
19
    <xsl:template match="oai:metadata">
20
        <!-- Only apply further templates to marc:record elements -->
21
        <!-- This prevents the identity transformation from outputting other non-MARC metadata formats -->
22
        <xsl:apply-templates select="//marc:record"/>
23
    </xsl:template>
24
25
    <!-- We need to create a MARCXML record from OAI records marked "deleted" to handle OAI deletions correctly in Koha -->
26
    <xsl:template match="oai:header[@status='deleted']">
27
        <xsl:element name="record" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
28
            xmlns="http://www.loc.gov/MARC21/slim">
29
            <xsl:attribute name="xsi:schemaLocation">http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd</xsl:attribute>
30
            <xsl:call-template name="add_oai"/>
31
        </xsl:element>
32
    </xsl:template>
33
34
    <!-- Identity transformation: this template copies attributes and nodes -->
35
    <xsl:template match="@* | node()">
36
        <!-- Create a copy of this attribute or node -->
37
        <xsl:copy>
38
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
39
            <xsl:apply-templates select="@* | node()" />
40
        </xsl:copy>
41
    </xsl:template>
42
43
44
    <xsl:template match="marc:record">
45
        <xsl:copy>
46
            <!-- Apply all relevant templates for all attributes and elements -->
47
            <xsl:apply-templates select="@* | node()"/>
48
49
            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
50
            <xsl:call-template name="add_oai"/>
51
52
            <!-- Newline -->
53
            <xsl:text>&#xa;</xsl:text>
54
        </xsl:copy>
55
    </xsl:template>
56
57
    <!-- Template for adding the OAI-PMH identifier as 024$a -->
58
    <xsl:template name="add_oai">
59
        <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
60
            <xsl:attribute name="ind1"><xsl:text>7</xsl:text></xsl:attribute>
61
            <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
62
            <xsl:attribute name="tag">024</xsl:attribute>
63
            <xsl:element name="subfield">
64
                <xsl:attribute name="code">a</xsl:attribute>
65
                <xsl:value-of select="/oai:record/oai:header/oai:identifier"/>
66
            </xsl:element>
67
            <xsl:element name="subfield">
68
                <xsl:attribute name="code">2</xsl:attribute>
69
                <xsl:text>uri</xsl:text>
70
            </xsl:element>
71
         </xsl:element>
72
    </xsl:template>
73
74
</xsl:stylesheet>
(-)a/svc/import_oai (+160 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
#
20
21
use Modern::Perl;
22
use XML::LibXML;
23
use URI;
24
use File::Basename;
25
26
use CGI qw ( -utf8 );
27
use C4::Auth qw/check_api_auth/;
28
use C4::Context;
29
use C4::ImportBatch;
30
use C4::Matcher;
31
use XML::Simple;
32
use C4::Biblio;
33
34
use Koha::OAI::Client::Record;
35
36
my $query = new CGI;
37
binmode STDOUT, ':encoding(UTF-8)';
38
39
my ($status, $cookie, $sessionID) = check_api_auth($query, { editcatalogue => 'edit_catalogue'} );
40
unless ($status eq "ok") {
41
    print $query->header(-type => 'text/xml', -status => '403 Forbidden');
42
    print XMLout({ auth_status => $status }, NoAttr => 1, RootName => 'response', XMLDecl => 1);
43
    exit 0;
44
}
45
46
my $xml;
47
if ($query->request_method eq "POST") {
48
    $xml = $query->param('xml');
49
}
50
if ($xml) {
51
    #TODO: You could probably use $query->Vars here instead...
52
    my %params = map { $_ => scalar $query->param($_) } $query->param;
53
    my $result = import_oai($xml, \%params );
54
    print $query->header(-type => 'text/xml');
55
    print XMLout($result, NoAttr => 1, RootName => 'response', XMLDecl => 1);
56
} else {
57
    print $query->header(-type => 'text/xml', -status => '400 Bad Request');
58
}
59
60
exit 0;
61
62
sub import_oai {
63
    my ($inxml, $params) = @_;
64
65
    #Create record object
66
    my $oai_record = Koha::OAI::Client::Record->new({
67
        xml_string => $inxml,
68
    });
69
    
70
    my $result = {};
71
    my $result_errors = [];
72
73
    my $filter      = delete $params->{filter}      || '';
74
    my $import_mode = delete $params->{import_mode} || '';
75
    my $framework   = delete $params->{framework}   || '';
76
    
77
    unless ($params->{comments}){
78
        $params->{comments} = "OAI-PMH import";
79
        if ($oai_record->{header_identifier}){
80
            $params->{comments} .= ": $oai_record->{header_identifier}";
81
        }
82
    }
83
84
    if (my $matcher_code = delete $params->{match}) {
85
        $params->{matcher_id} = C4::Matcher::GetMatcherId($matcher_code);
86
    }
87
88
    my $batch_id = GetWebserviceBatchId($params);
89
    #FIXME: Use the batch_id to create a more useful filename in the import_batches table...
90
    unless ($batch_id) {
91
        $result->{'status'} = "failed";
92
        $result->{'error'} = "Batch create error";
93
        return $result;
94
    }
95
96
    #Source a default XSLT to use for filtering
97
    my $htdocs  = C4::Context->config('intrahtdocs');
98
    my $theme   = C4::Context->preference("template");
99
    #FIXME: This doesn't work for UNIMARC!
100
    my $xslfilename = "$htdocs/$theme/en/xslt/OAI2MARC21slim.xsl";
101
102
    #FIXME: There's a better way to do these filters...
103
    if ($filter){
104
        my $filter_uri = URI->new($filter);
105
        if ($filter_uri){
106
            my $scheme = $filter_uri->scheme;
107
            if ($scheme && $scheme eq "file"){
108
                my $path = $filter_uri->path;
109
                #Filters may theoretically be .xsl or .pm files
110
                my($filename, $dirs, $suffix) = fileparse($path,(".xsl",".pm"));
111
                if ($suffix && $suffix eq ".xsl"){
112
                    #If this new path exists, change the filter XSLT to it
113
                    if ( -f $path ){
114
                        $xslfilename = $path;
115
                    }
116
                }
117
            }
118
        }
119
    }
120
121
    #Get matching rule matcher
122
    my $matcher = C4::Matcher->new($params->{record_type} || 'biblio');
123
    $matcher = C4::Matcher->fetch($params->{matcher_id});
124
125
    
126
127
    #Filter OAI-PMH into MARCXML
128
    my $filtered = $oai_record->filter({
129
        filter => $xslfilename,
130
    });
131
132
    if (!$filtered){
133
        push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
134
    }
135
136
    my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
137
        matcher => $matcher,
138
        import_batch_id => $batch_id,
139
        import_mode => $import_mode,
140
        framework => $framework,
141
    });
142
    if (@$errors){
143
        push(@$result_errors,@$errors);
144
    }
145
146
    $oai_record->save_to_database();
147
148
    $result->{'match_status'} = $match_status;
149
    $result->{'import_batch_id'} = $batch_id;
150
    $result->{'koha_record_numbers'} = $koha_record_numbers;
151
152
    if ($import_status && $import_status eq "ok"){
153
        $result->{'status'} = "ok";
154
    } else {
155
        $result->{'status'} = "failed";
156
        $result->{'errors'} = {error => $result_errors};
157
    }
158
159
    return $result;
160
}
(-)a/t/Import/bib-deleted.xml (+7 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns="http://www.openarchives.org/OAI/2.0/">
3
  <header status="deleted">
4
    <identifier>oai:koha-community.org:5000</identifier>
5
    <datestamp>2015-12-22T18:46:29Z</datestamp>
6
  </header>
7
</record>
(-)a/t/Import/bib-oaidc.xml (+19 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns="http://www.openarchives.org/OAI/2.0/">
3
  <header>
4
    <identifier>oai:koha-community.org:5000</identifier>
5
    <datestamp>2015-12-21T18:46:29Z</datestamp>
6
  </header>
7
  <metadata>
8
    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
9
        <oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd">
10
          <dc:title>Everything you never wanted to know about OAI-PMH : a primer /</dc:title>
11
          <dc:creator>
12
            Cook, David
13
          </dc:creator>
14
          <dc:type>text</dc:type>
15
          <dc:language>eng</dc:language>
16
        </oai_dc:dc>
17
    </metadata>
18
  </metadata>
19
</record>
(-)a/t/Import/bib.xml (+25 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<record xmlns="http://www.openarchives.org/OAI/2.0/">
3
  <header>
4
    <identifier>oai:koha-community.org:5000</identifier>
5
    <datestamp>2015-12-21T18:46:29Z</datestamp>
6
  </header>
7
  <metadata>
8
    <metadata xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
9
      <record xmlns="http://www.loc.gov/MARC21/slim" type="Bibliographic">
10
        <leader>01005cam a22003377a 4500</leader>
11
        <controlfield tag="001">123456789</controlfield>
12
        <controlfield tag="005">20151221185246.0</controlfield>
13
        <controlfield tag="008">010203s2004    nyu           000 0 eng u</controlfield>
14
        <datafield ind1="1" tag="100" ind2=" ">
15
          <subfield code="a">Cook, David</subfield>
16
        </datafield>
17
        <datafield ind1="1" ind2="0" tag="245">
18
          <subfield code="a">Everything you never wanted to know about OAI-PMH :</subfield>
19
          <subfield code="b">a primer /</subfield>
20
          <subfield code="c">David Cook</subfield>
21
        </datafield>
22
      </record>
23
    </metadata>
24
  </metadata>
25
</record>
(-)a/tools/manage-oai-import.pl (-1 / +206 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2016 Prosentient Systems
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
use Modern::Perl;
21
use POSIX qw//;
22
23
use Koha::Database;
24
25
use C4::Auth;
26
use C4::Output;
27
use C4::Koha;
28
use C4::Context;
29
use C4::Matcher;
30
use Koha::OAI::Client::Record;
31
32
use C4::ImportBatch;
33
34
my $script_name = "/cgi-bin/koha/tools/manage-oai-import.pl";
35
36
my $input = new CGI;
37
my $op = $input->param('op') || 'list';
38
my $page_number = $input->param('page') && $input->param('page') =~ /^\d+$/ ? $input->param('page') : 1; #Only permit numeric parameter
39
40
my $import_oai_id = $input->param('import_oai_id');
41
#my $results_per_page = $input->param('results_per_page') || 25;
42
43
44
45
my ($template, $loggedinuser, $cookie) =
46
    get_template_and_user({template_name => "tools/manage-oai-import.tt",
47
        query => $input,
48
        type => "intranet",
49
        authnotrequired => 0,
50
        flagsrequired => {tools => 'manage_staged_marc'},
51
        debug => 1,
52
    });
53
54
my $schema = Koha::Database->new()->schema();
55
my $resultset = $schema->resultset('ImportOai');
56
57
58
if ($import_oai_id){
59
    my $import_oai_record = $resultset->find($import_oai_id);
60
    $template->param(
61
        oai_record => $import_oai_record,
62
    );
63
64
    if ($op eq "view_record" && $import_oai_id){
65
        $template->param(
66
            view_record => 1,
67
            import_oai_id => $import_oai_id,
68
        );
69
    }
70
71
    if ($op eq "retry" && $import_oai_record){
72
        my $result_errors = [];
73
74
        my $oai_record = Koha::OAI::Client::Record->new({
75
            xml_string => $import_oai_record->metadata,
76
        });
77
78
        my $filtered = $oai_record->filter({
79
            filter => $import_oai_record->filter,
80
        });
81
        if (!$filtered){
82
            push(@$result_errors, { type => 'filter_failed', error_msg => '', record_id => '', }) ;
83
        }
84
85
        my $import_batch_id = $import_oai_record->import_batch_id;
86
        if ($import_batch_id){
87
            my $import_batch_rs = $schema->resultset('ImportBatch');
88
            my $import_batch = $import_batch_rs->find($import_batch_id);
89
            my $matcher_id = $import_batch->matcher_id;
90
91
            my $record_type = $import_batch->record_type;
92
            $template->param(
93
                record_type => $record_type,
94
            );
95
96
97
            #my $matcher = C4::Matcher->new($record_type || 'biblio');
98
            my $matcher = C4::Matcher->fetch($matcher_id);
99
100
101
            #FIXME
102
            my $import_mode = "direct";
103
            #FIXME
104
            my $framework = "";
105
106
            #Reset the batch before re-trying the import
107
            C4::ImportBatch::CleanBatch($import_batch_id);
108
109
            my ($import_status, $match_status, $koha_record_numbers, $errors) = $oai_record->import_record({
110
                matcher => $matcher,
111
                import_batch_id => $import_batch_id,
112
                import_mode => $import_mode,
113
                framework => $framework,
114
            });
115
            if (@$errors){
116
                push(@$result_errors,@$errors);
117
            }
118
119
            if ($import_status){
120
                if ($import_status eq 'ok'){
121
                    $import_oai_record->status("ok");
122
                    $import_oai_record->update();
123
                } else {
124
                    $template->param(
125
                        import_status => $import_status,
126
                        errors => $result_errors,
127
                        retry => 1,
128
                        import_oai_id => $import_oai_id,
129
                    );
130
                }
131
            }
132
        }
133
    }
134
}
135
136
137
138
$template->param(
139
    script_name => $script_name,
140
);
141
142
if ($op && $op eq "list"){
143
    #NOTE: It would be preferable if we centralized server-side paging code with generic functions...
144
145
    #Get grand total in the database
146
    my $total_rows = $resultset->count;
147
148
    my $number_of_rows = 10;
149
    my $number_of_pages = POSIX::ceil($total_rows / $number_of_rows);
150
151
    if ($page_number > 1){
152
        $template->{VARS}->{ page_first } = 1;
153
    }
154
    if ($number_of_pages > 1 && $page_number != $number_of_pages){
155
        $template->{VARS}->{ page_last } = $number_of_pages;
156
    }
157
158
    #Do the search and define a limit
159
    my $results = $resultset->search(
160
        undef,
161
        {
162
            rows => $number_of_rows,
163
            order_by => { -desc => 'last_modified' },
164
        },
165
    );
166
167
    my $page = $results->page($page_number);
168
    my $current_page_rows = $page->count();
169
170
    if ($current_page_rows){
171
        if ($page_number == 1) {
172
            #Can't page previous to the first page
173
            if ( $current_page_rows == $total_rows ){
174
                #Can't page past the total count
175
            } else {
176
                #Signal that there's another page
177
                $template->{VARS}->{ page_next } = $page_number + 1;
178
            }
179
        } else {
180
181
            #Signal that there's a previous page
182
            $template->{VARS}->{ page_previous } = $page_number - 1;
183
184
            #Total rows of all previous pages
185
            my $previous_pages_rows = ( $page_number - 1 ) * $number_of_rows;
186
187
            #If current and past rows are less than total rows, there must still be more to show
188
            if ( ($current_page_rows + $previous_pages_rows) < $total_rows){
189
                $template->{VARS}->{ page_next } = $page_number + 1;
190
            }
191
192
        }
193
    }
194
195
    $template->param(
196
        page_number => $page_number,
197
        script_name => $script_name,
198
        oai_records => $page,
199
    );
200
}
201
202
203
204
205
206
output_html_with_http_headers $input, $cookie, $template->output;

Return to bug 10662