View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

(-)a/Koha/OAI/Client/Importer.pm (+316 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Importer;
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, see <http://www.gnu.org/licenses>.
18
19
use Modern::Perl;
20
21
use Koha::Database;
22
use Koha::XSLT_Handler;
23
use C4::Context;
24
use C4::Languages;
25
use C4::XSLT;
26
use C4::Biblio;
27
28
use parent qw(Class::Accessor);
29
__PACKAGE__->mk_accessors( qw (xslt_handler verbosity) );
30
31
=head1 NAME
32
33
Koha::OAI::Importer - Imports records, previously harvested via OAI-PMH, into Koha
34
35
=head1 SYNOPSIS
36
37
  my $oai_importer = Koha::OAI::Importer->new({
38
    verbosity => $verbose,
39
  });
40
  $oai_importer->import_records();
41
42
=head1 DESCRIPTION
43
44
An OAI Importer is an object which retrieves records, previously harvested
45
via OAI-PMH, and determines whether they should be added as new records to the
46
catalogue or if they should update existing catalogue records.
47
48
After this determination, the record's metadata is run through a XSLT, which
49
is configured on a per server basis, and then handed over to Koha's internal APIs.
50
51
The record in the OAI harvesting queue is then updated to reflect the
52
action taken and the biblionumber to which it is linked.
53
54
=head1 METHODS
55
56
=head2 new
57
58
The Importer constructor which sets the 'verbosity' and 'xslt_handler'
59
for the object.
60
61
The 'verbosity' should be set by calling script.
62
63
The 'xslt_handler' could also be set by a calling script, but defaults
64
to a new Koha::XSLT_Handler object with silenced warnings, so it is
65
not too noisy.
66
67
=cut
68
69
sub new {
70
    my ($class, $args) = @_;
71
    $args = {} unless defined $args;
72
    $args->{xslt_handler} = Koha::XSLT_Handler->new({print_warns => 0,}) unless defined $args->{xslt_handler};
73
    $args->{verbosity} = 0 unless defined $args->{verbosity};
74
    return bless ($args, $class);
75
}
76
77
sub delete_record {
78
    my ( $self, $record ) = @_;
79
80
    my ($action, $system_number) = $self->determine_action_and_system_number({
81
        identifier => $record->identifier,
82
        repository_id => $record->repository_id,
83
    });
84
    my $error = C4::Biblio::DelBiblio($system_number);
85
    if ($error){
86
        die $error;
87
    } else {
88
        $record->set({
89
            system_number => $system_number,
90
            action => "deleted",
91
        });
92
        $record->store();
93
    }
94
}
95
96
=head2 import_record
97
98
This is the method doing the heavy lifting. It runs the other
99
methods for determining what to do with the record and if it is
100
linked to an existing record, it transforms the record via XSLT,
101
and it ultimately adds or updates the record as need be.
102
103
=cut
104
105
sub import_record {
106
    my ( $self, $record ) = @_;
107
108
    #Step one: determine action (i.e. determine if a record with this identifier from this repository has been successfully added before)
109
    my ($action, $system_number) = $self->determine_action_and_system_number({
110
        identifier => $record->identifier,
111
        repository_id => $record->repository_id,
112
    });
113
114
    #Step two: transform metadata (if necessary) && add/update in Koha
115
    if ( ($action eq 'add') || ($action eq 'update') ){
116
        my $schema = Koha::Database->new()->schema();
117
118
        #Lookup the repository so we can get the xslt_path and frameworkcode
119
        my $repository = $schema->resultset('OaiHarvestRepository')->find($record->repository_id);
120
121
        my $metadata = $record->metadata;
122
        if ($metadata && $repository->xslt_path){
123
            eval {
124
                #If there is a XSLT path, try to transform metadata
125
                my $transformed_metadata = $self->transform_metadata({
126
                    xslt_path => $repository->xslt_path,
127
                    metadata => $metadata,
128
                    metadata_prefix => $record->metadata_prefix,
129
                    identifier => $record->identifier,
130
                });
131
                if ($transformed_metadata){
132
                    $metadata = $transformed_metadata;
133
                }
134
            };
135
            if ($@){
136
                warn "There was a non-fatal problem transforming the metadata during import: $@";
137
            }
138
        }
139
        if ($metadata){
140
            if ($record->record_type eq 'biblio'){
141
                my ( $biblionumber, $biblioitemnumber );
142
                #create MARC::Record object
143
                my $marc_record = MARC::Record::new_from_xml( $metadata, "utf8", C4::Context->preference('marcflavour') );
144
                if ($marc_record){
145
                    if ($action eq 'add'){
146
                        #Add record to Koha
147
                        ($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode);
148
                    } elsif ($action eq 'update'){
149
                        #Update record in Koha
150
                        eval {C4::Biblio::ModBiblio( $marc_record,$system_number,$repository->frameworkcode)};
151
                        if ($@){
152
                            my $deleted_biblio = $schema->resultset('Deletedbiblio')->find($system_number);
153
                            my $existing_biblio = $schema->resultset('Biblio')->find($system_number);
154
                            if ( ( $deleted_biblio ) || (!$deleted_biblio && !$existing_biblio) ){
155
                                die "It appears the original record referenced by biblionumber $system_number has been deleted.";
156
                                #FIXME: What should happen in this situation? You can't re-add the record without first removing
157
                                #the previous oai_harvest entries or there will be errors...
158
                                #($biblionumber,$biblioitemnumber) = C4::Biblio::AddBiblio($marc_record,$repository->frameworkcode);
159
                                #warn "Re-adding the incoming record as a new bibliographic record (biblionumber $system_number).";
160
                                #$action = "re-add";
161
                            } else {
162
                                die $@;
163
                            }
164
                        }
165
                    }
166
                }
167
                if ($biblionumber){
168
                    $system_number = $biblionumber;
169
                }
170
            }
171
            #TODO: Add handling for 'holdings' and 'auth'
172
        }
173
        if ($action && $system_number){
174
            #Update 'action' and 'system_number' columns for this record row
175
            $record->set({
176
                action => $action,
177
                system_number => $system_number,
178
            });
179
            $record->store();
180
        } else {
181
            die "Error importing record! action: ($action);";
182
        }
183
    }
184
}
185
186
=head 2 determine_action_and_system_number
187
188
This method checks the database to see if a record from the same
189
OAI-PMH server with the same identifier has already been added
190
into Koha. If it hasn't, it should be added. If it has, it should
191
update the existing record.
192
193
=cut
194
195
sub determine_action_and_system_number {
196
    my ( $self, $args ) = @_;
197
    my ( $action, $system_number );
198
    if ( $args->{identifier} && $args->{repository_id} ){
199
        my $schema = Koha::Database->new()->schema();
200
201
        #NOTE: $search_results is a DBIx::Class::ResultSet object
202
        my $search_results = $schema->resultset('OaiHarvest')->search(
203
            {
204
                -and => [
205
                            identifier => $args->{identifier},
206
                            repository_id => $args->{repository_id},
207
                            system_number => {'!=', undef},
208
                            -or => [
209
                                action => 'add',
210
                                action => 'update',
211
                            ],
212
                        ],
213
            },
214
            {
215
              columns => [ qw/system_number/ ], #Only retrieve system_number
216
              group_by => [ qw/system_number/ ], #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number
217
            }
218
        );
219
220
        #NOTE: The DBIx::Class::ResultSet class has overloaded operators,
221
        #which means a "count()" method is run whenever an object is called in a
222
        #numerical context (like below). It will also always return 1 in a boolean context.
223
        if ($search_results == 0){
224
            $action = 'add';
225
        } elsif ($search_results == 1){
226
            $action = 'update';
227
            while( my $result = $search_results->next ) {
228
                $system_number = $result->system_number;
229
            }
230
        } else {
231
            if ($search_results && $search_results > 1){
232
                die "More than one system_number matched the identifier " . $args->{identifier} . " with a repository_id of " . $args->{repository_id} . ".";
233
            } else {
234
                die "Unknown error scanning database for previous entries.";
235
            }
236
        }
237
    }
238
    return ($action,$system_number);
239
}
240
241
=head2 transform_metadata
242
243
This method takes a record's XML metadata and runs it through a XSLT
244
(using a Koha::XSLT_Handler object). The purpose of this is to remove
245
undesirable fields or to add desired ones.
246
247
=cut
248
249
sub transform_metadata {
250
    my ( $self, $args ) = @_;
251
    my $transformed_metadata;
252
253
    if ($self->xslt_handler){
254
        if ( ($args->{xslt_path}) && ($args->{metadata_prefix}) && ($args->{metadata}) && ($args->{identifier}) ){
255
256
            my $xslt = _resolve_xslt_path($args->{xslt_path}, $args->{metadata_prefix});
257
258
            #If there is a XSLT file, we try to transform the data
259
            if ($xslt){
260
                require XML::LibXSLT;
261
                #NOTE: You can use "register_function()" here to provide extension functions to the XSLT
262
                #NOTE: For parameters, you must wrap XML::LibXSLT::xpath_to_string() around each key => value pair within a hashref
263
                #{ XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), XML::LibXSLT::xpath_to_string(test => "tester"), }
264
                $transformed_metadata = $self->xslt_handler->transform({
265
                    xml => $args->{metadata},
266
                    file => $xslt,
267
                    parameters => { XML::LibXSLT::xpath_to_string(identifier => $args->{identifier}), },
268
                });
269
                if ( $self->xslt_handler->err ){
270
                    die $self->xslt_handler->errstr;
271
                }
272
                if (! $transformed_metadata){
273
                    die "Unable to transform metadata";
274
                }
275
            }
276
        }
277
    }
278
279
    return $transformed_metadata;
280
}
281
282
=head2 _resolve_xslt_path
283
284
An internal sub for turning magical values into actual filepaths to a XSLT.
285
286
=cut
287
288
sub _resolve_xslt_path {
289
    my ( $xslfilename, $metadata_prefix ) = @_;
290
    if ( $xslfilename ){
291
        if ( $xslfilename =~ /^\s*"?default"?\s*$/i ) {
292
            if ($metadata_prefix =~ "marc"){
293
                #If the repository has a default XSLT and harvests MARCXML:
294
                my $htdocs  = C4::Context->config('intrahtdocs');
295
                my $theme   = C4::Context->preference("template");
296
                my $lang = C4::Languages::getlanguage();
297
                my $xslfile = C4::Context->preference('marcflavour') .
298
                                   "slimFromOAI.xsl";
299
300
                $xslfilename = C4::XSLT::_get_best_default_xslt_filename($htdocs, $theme, $lang, $xslfile);
301
                if ( ! -f $xslfilename ){
302
                    #If the default XSLT doesn't exist, nullify this variable
303
                    $xslfilename = undef;
304
                }
305
            } else {
306
                #If the repository has a default XSLT and doesn't harvest marc:
307
                #We return a null because currently we only provide default XSLTs for MARC21 marcxml
308
                $xslfilename = undef;
309
            }
310
        }
311
    }
312
    return $xslfilename;
313
}
314
315
316
1;
(-)a/Koha/OAI/Client/Record.pm (+61 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Record;
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use Carp;
22
23
use base qw(Koha::Object);
24
use C4::Biblio;
25
26
=head1 NAME
27
28
Koha::OAI::Client::Record -
29
30
=head1 API
31
32
=head2 Class Methods
33
34
=cut
35
36
sub delete {
37
    my ( $self ) = @_;
38
    if ($self->record_type eq 'biblio' && $self->system_number){
39
        my $error = C4::Biblio::DelBiblio($self->system_number);
40
        if ($error){
41
            die $error;
42
        }
43
    }
44
    $self->SUPER::delete();
45
}
46
47
=head3 type
48
49
=cut
50
51
sub type {
52
    return 'OaiHarvest';
53
}
54
55
=head1 AUTHOR
56
57
David Cook <dcook@prosentient.com.au>
58
59
=cut
60
61
1;
(-)a/Koha/OAI/Client/Records.pm (+58 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Records;
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use Carp;
22
23
use base qw(Koha::Objects);
24
use Koha::OAI::Client::Record;
25
26
=head1 NAME
27
28
Koha::OAI::Client::Records -
29
30
=head1 API
31
32
=head2 Class Methods
33
34
=cut
35
36
=head3 type
37
38
=cut
39
40
sub type {
41
    return 'OaiHarvest';
42
}
43
44
=head3 object_class
45
46
=cut
47
48
sub object_class {
49
    return 'Koha::OAI::Client::Record';
50
}
51
52
=head1 AUTHOR
53
54
David Cook <dcook@prosentient.com.au>
55
56
=cut
57
58
1;
(-)a/Koha/OAI/Client/Repositories.pm (+62 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Repositories;
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
22
use Carp;
23
24
use Koha::Database;
25
26
use Koha::OAI::Client::Repository;
27
28
use base qw(Koha::Objects);
29
30
=head1 NAME
31
32
Koha::OAI::Client::Repositories -
33
34
=head1 API
35
36
=head2 Class Methods
37
38
=cut
39
40
=head3 type
41
42
=cut
43
44
sub type {
45
    return 'OaiHarvestRepository';
46
}
47
48
=head3 object_class
49
50
=cut
51
52
sub object_class {
53
    return 'Koha::OAI::Client::Repository';
54
}
55
56
=head1 AUTHOR
57
58
David Cook <dcook@prosentient.com.au>
59
60
=cut
61
62
1;
(-)a/Koha/OAI/Client/Repository.pm (+418 lines)
Line 0 Link Here
1
package Koha::OAI::Client::Repository;
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use Carp;
22
use POSIX qw(strftime);
23
use HTTP::OAI;
24
use URI;
25
use DateTime;
26
use DateTime::Format::Strptime;
27
use base qw(Koha::Object);
28
29
use Koha::OAI::Client::Records;
30
31
=head1 NAME
32
33
Koha::OAI::Client::Repository -
34
35
=head1 API
36
37
=head2 Class Methods
38
39
=cut
40
41
=head3 test_settings
42
43
=cut
44
45
sub test_settings {
46
    my ( $self ) = @_;
47
    my $errors = {};
48
    my $harvester = $self->harvester();
49
    if ($harvester){
50
        my $identify = $harvester->Identify;
51
        if ($identify->is_success){
52
53
            #Test Granularity
54
            my $granularity_setting = $self->datetime_granularity;
55
            if ($granularity_setting eq "YYYY-MM-DDThh:mm:ssZ"){
56
                my $actual_granularity = $identify->granularity;
57
                if ($granularity_setting ne $actual_granularity){
58
                    $errors->{second_granularity_not_supported} = 1;
59
                }
60
            }
61
62
            #Test Set
63
            my $set_setting = $self->opt_set;
64
            if ($set_setting){
65
                my $sets = $harvester->ListSets();
66
                my $matched_set;
67
                while ( my $set = $sets->next ){
68
                    if ($set_setting eq $set->setSpec){
69
                        $matched_set = 1;
70
                        last;
71
                    }
72
                }
73
                if ( ! $matched_set ){
74
                    $errors->{set_does_not_exist} = 1;
75
                }
76
            }
77
78
            #Test Metadata prefix
79
            my $metadata_prefix_setting = $self->metadata_prefix;
80
            if ($metadata_prefix_setting){
81
                my $metadata_formats = $harvester->ListMetadataFormats();
82
                my $matched_format;
83
                while ( my $metadata_format = $metadata_formats->next ){
84
                    if ($metadata_prefix_setting eq $metadata_format->metadataPrefix){
85
                        $matched_format = 1;
86
                        last;
87
                    }
88
                }
89
                if ( ! $matched_format ){
90
                    $errors->{metadata_prefix_does_not_exist} = 1;
91
                }
92
            }
93
        } else {
94
            if ($identify->is_error()){
95
                foreach my $error ($identify->errors){
96
                    if ($error->code =~ /^404$/){
97
                        $errors->{url_not_found} = 1;
98
                    } elsif ($error->code =~ /^401$/){
99
                        $errors->{failed_authentication} = 1;
100
                    } else {
101
                        $errors->{generic_identify_error} = 1;
102
                    }
103
                }
104
            } else {
105
                #This should never happen, but you never know...
106
                $errors->{generic_identify_error} = 1;
107
            }
108
        }
109
    } else {
110
        $errors->{no_harvester} = 1;
111
    }
112
    return $errors;
113
}
114
115
116
=head3 validate
117
118
=cut
119
120
sub validate {
121
    my ( $self ) = @_;
122
    my $errors = {};
123
    if ($self->base_url){
124
        my $uri = URI->new($self->base_url);
125
        if ($uri){
126
            my $reference = ref $uri;
127
            if ( ! grep { $reference eq $_ } ("URI::http","URI::https")){
128
                $errors->{base_url}->{invalid} = 1;
129
            }
130
        }
131
    } elsif (! defined $self->base_url){
132
        #base_url is undefined
133
        $errors->{base_url}->{required} = 1;
134
    }
135
    if ( ! defined $self->metadata_prefix ){
136
        $errors->{metadata_prefix}->{required} = 1;
137
    }
138
    return $errors;
139
}
140
141
=head3 find_original_system_number
142
143
=cut
144
145
sub find_original_system_number {
146
    my ( $self, $args ) = @_;
147
148
    my $original_system_number;
149
    my $metadata = $args->{metadata};
150
    if ($metadata){
151
        my $original_system_field = $self->original_system_field; #e.g. 001 or 999$c
152
        if ($original_system_field){
153
            #Select root element (e.g. <metadata>)
154
            my $root = $metadata->documentElement();
155
            #Select the first child of root (ie the original metadata record) (eg <record>)
156
            my $original_metadata = $root->firstChild;
157
158
            #Check to see if there is a MARC namespace
159
            my $marcxml_prefix = $original_metadata->lookupNamespacePrefix("http://www.loc.gov/MARC21/slim");
160
            if (defined $marcxml_prefix){
161
                my ($tag,$code) = split(/\$/,$original_system_field); #e.g. split 999$c into 999 and c || split 001 into 001
162
                my $xpc = XML::LibXML::XPathContext->new;
163
                $xpc->registerNs('marc', 'http://www.loc.gov/MARC21/slim');
164
                my $fields = $xpc->find(qq(//marc:record/node()[\@tag="$tag"]),$original_metadata);
165
                if ($fields){
166
                    #Use the first field found
167
                    my $field = $fields->get_node(1);
168
                    if ($field){
169
                        my $node_name = $field->nodeName;
170
                        if ($node_name eq 'controlfield'){
171
                            #If it's a controlfield, we can use the text without drilling down
172
                            $original_system_number = $field->textContent;
173
                        } elsif ($node_name eq 'datafield'){
174
                            #If it's a datafield, we need to drill down into the subfields
175
                            if ($code){
176
                                my $subfields = $field->find(qq(node()[\@code="$code"]));
177
                                if ($subfields){
178
                                    #Use the first subfield found
179
                                    my $subfield = $subfields->get_node(1);
180
                                    if ($subfield){
181
                                        $original_system_number = $subfield->textContent;
182
                                    }
183
184
                                }
185
                            }
186
                        }
187
                    }
188
                }
189
            }
190
        }
191
    }
192
    return $original_system_number;
193
}
194
195
=head3 queue_record
196
197
=cut
198
199
sub queue_record {
200
    my ( $self, $args ) = @_;
201
    my $record = $args->{record};
202
    my $stylesheet = $args->{stylesheet};
203
204
    my $action = "to_import";
205
    if ( $record->is_deleted() ){
206
        my $already_deleted_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier, action => "deleted" });
207
        my $identified_rows = Koha::OAI::Client::Records->search({ identifier => $record->identifier });
208
        if ($identified_rows->count() > 0 && $already_deleted_rows->count() == 0){
209
            #Only queue records "to_delete", if the record has been previously harvested
210
            $action = "to_delete";
211
        } else {
212
            return;
213
        }
214
    }
215
216
    my $metadata = $record->metadata ? $record->metadata->dom : '';
217
    my $original_system_number;
218
    if ($metadata){
219
        if ($stylesheet){
220
            #Use a stylesheet to strip the OAI-PMH metadata XML wrapper
221
            eval {
222
                my $result = $stylesheet->transform($metadata);
223
                if ($result){
224
                    $metadata = $result;
225
                }
226
            };
227
            if ($@){
228
                warn "Problem transforming harvested metadata with XSLT: $@";
229
            }
230
        }
231
232
        $original_system_number = $self->find_original_system_number({metadata => $metadata});
233
    }
234
235
    my $entry = Koha::OAI::Client::Records->_resultset->find_or_new({
236
        repository_id => $self->repository_id, #Internal ID for the OAI repository
237
        action => $action,
238
        identifier => $record->identifier, #External ID for the OAI record
239
        datestamp => $record->datestamp, #External datestamp for the OAI record
240
        metadata_prefix => $self->metadata_prefix, #External metadataPrefix for the OAI record
241
        metadata => $metadata ? $metadata->toString(1) : undef,
242
        record_type => $self->record_type,
243
        original_system_number => $original_system_number,
244
    },
245
    { key => "harvest_identifier_datestamp" }
246
    );
247
    if( !$entry->in_storage ) {
248
      $entry->insert;
249
    }
250
}
251
252
sub _format_datetime {
253
    my ( $self, $args ) = @_;
254
    my $datetime_string = $args->{datetime_string};
255
    my $formatted_string;
256
    if ($datetime_string && $datetime_string ne "0000-00-00 00:00:00"){
257
        my $strp = DateTime::Format::Strptime->new(
258
            pattern   => '%F %T',
259
        );
260
        my $dt = $strp->parse_datetime($datetime_string);
261
        if ($dt){
262
            if ($self->datetime_granularity && $self->datetime_granularity eq 'YYYY-MM-DDThh:mm:ssZ'){
263
                $formatted_string = $dt->strftime("%FT%TZ");
264
            } else {
265
                $formatted_string = $dt->strftime("%F");
266
            }
267
        }
268
    }
269
    return $formatted_string;
270
}
271
272
sub harvester {
273
    my ( $self ) = @_;
274
    my $harvester;
275
    if ($self->base_url){
276
        $harvester = new HTTP::OAI::Harvester( baseURL => $self->base_url );
277
        #Use basic http authentication credentials if they're set
278
       # if ($self->basic_realm && $self->basic_username && $self->basic_password){
279
            #Decompose the URI in order to register the basic authentication credentials
280
            my $uri = URI->new($self->base_url);
281
            my $host = $uri->host;
282
            my $port = $uri->port;
283
            $harvester->credentials($host.":".$port, $self->basic_realm, $self->basic_username, $self->basic_password);
284
       # }
285
    }
286
    return $harvester;
287
}
288
289
=head3 query_repository
290
291
=cut
292
293
sub query_repository {
294
    my ( $self ) = @_;
295
    my $response;
296
    my $h = $self->harvester();
297
    if ($h){
298
        my $opt_from = $self->opt_from;
299
        my $opt_until = $self->opt_until;
300
        foreach my $datetime ($opt_from,$opt_until){
301
            #Format the datetime strings from the database into the format and granularity
302
            #expected by the target OAI-PMH server
303
            $datetime = $self->_format_datetime({datetime_string => $datetime,});
304
        }
305
306
        #Issue the request to the OAI-PMH server
307
        $response = $h->ListRecords(
308
            metadataPrefix => $self->metadata_prefix,
309
            from => $opt_from,
310
            until => $opt_until,
311
            set => $self->opt_set,
312
        );
313
    }
314
    return $response;
315
}
316
317
=head3 harvest
318
319
=cut
320
321
sub harvest {
322
    my ( $self, $args ) = @_;
323
324
    my $verbose = $args->{verbose} // '';
325
    my $stylesheet = $args->{stylesheet} // '';
326
    $verbose && print "Harvesting records from ".$self->base_url."\n";
327
328
    #Record the exact time before sending the first OAI-PMH request, and set "opt_from" to this after the harvest is finished
329
    my $new_from_date = strftime "%Y-%m-%d %H:%M:%S", localtime;
330
    #Send a request to the OAI-PMH repository
331
    my $oai_response = $self->query_repository();
332
    if ($oai_response){
333
        if( $oai_response->is_error ){
334
            warn "responseDate => " . $oai_response->responseDate . "\n";
335
            warn "requestURL => " . $oai_response->requestURL . "\n";
336
            warn "Error harvesting: " . $oai_response->message . "\n";
337
        } else {
338
            while( my $rec = $oai_response->next ) {
339
                $verbose && print $rec->identifier."\n";
340
                #Queue each record, so a different script can process them in Koha
341
                $self->queue_record({record => $rec, stylesheet => $stylesheet, });
342
            }
343
            #Update opt_from after a successful harvest
344
            $self->opt_from($new_from_date);
345
            $verbose && print "Updating `opt_from` to $new_from_date \n";
346
            $self->store();
347
        }
348
    }
349
}
350
351
#FIXME
352
=head3 reharvest
353
354
=cut
355
356
sub reharvest {
357
    my ( $self, $args ) = @_;
358
    my $verbose = $args->{verbose} // '';
359
    eval {
360
        $self->delete_harvest({ verbose => $verbose, });
361
    };
362
    if ($@){
363
        warn $@;
364
    } else {
365
        $self->harvest({ verbose => $verbose, });
366
    }
367
}
368
369
#FIXME
370
=head3 delete_harvest
371
372
=cut
373
374
sub delete_harvest {
375
    my ( $self ) = @_;
376
377
    my $count_of_records = 0;
378
    my $count_of_deleted_records = 0;
379
    my $problem_records = [];
380
381
    if ($self->repository_id){
382
        my $records = Koha::OAI::Client::Records->search({ "repository_id" => $self->repository_id });
383
        while (my $record = $records->next){
384
            #Increment overall count
385
            $count_of_records++;
386
            eval {
387
                my $is_deleted = $record->delete();
388
                if ($is_deleted){
389
                    $count_of_deleted_records++;
390
                }
391
            };
392
            if ($@){
393
                push(@$problem_records,$record);
394
            }
395
        }
396
    }
397
398
    if ($count_of_deleted_records < $count_of_records){
399
        #Not all of this repository's records could be deleted. Manual intervention may be required to remove items
400
        die $problem_records;
401
    }
402
}
403
404
=head3 type
405
406
=cut
407
408
sub type {
409
    return 'OaiHarvestRepository';
410
}
411
412
=head1 AUTHOR
413
414
David Cook <dcook@prosentient.com.au>
415
416
=cut
417
418
1;
(-)a/admin/oai_client.pl (+220 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it
8
# under the terms of the GNU General Public License as published by
9
# the Free Software Foundation; either version 3 of the License, or
10
# (at your option) any later version.
11
#
12
# Koha is distributed in the hope that it will be useful, but
13
# WITHOUT ANY WARRANTY; without even the implied warranty of
14
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
# GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
20
=head1 NAME
21
22
oai_client.pl
23
24
=head1 DESCRIPTION
25
26
Admin page to manage Koha's target OAI-PMH servers
27
28
=cut
29
30
use Modern::Perl;
31
use CGI qw ( -utf8 );
32
use C4::Auth;
33
use C4::Output;
34
use C4::Koha qw/GetFrameworksLoop/;
35
use Koha::OAI::Client::Repository;
36
use Koha::OAI::Client::Repositories;
37
use Koha::OAI::Client::Records;
38
39
my $input = new CGI;
40
41
my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( {
42
    template_name   => 'admin/oai_client.tt',
43
    query           => $input,
44
    type            => 'intranet',
45
    authnotrequired => 0,
46
    flagsrequired   => { 'parameters' => 'parameters_remaining_permissions' },
47
} );
48
49
my $op = $input->param('op');
50
my $test_settings = $input->param('test_settings');
51
my $repository_id = $input->param('repository_id');
52
53
my $base_url = $input->param('base_url') || undef;
54
my $metadata_prefix = $input->param('metadata_prefix') || undef;
55
my $opt_from = $input->param('opt_from') || undef;
56
my $opt_until = $input->param('opt_until') || undef;
57
my $opt_set = $input->param('opt_set') || undef;
58
my $active = $input->param('active');
59
my $xslt_path = $input->param('xslt_path') || undef;
60
my $frameworkcode = $input->param('frameworkcode'); #This can be empty as the default framework is an empty value
61
my $comments = $input->param('comments') || undef;
62
my $basic_username = $input->param('basic_username') || undef;
63
my $basic_password = $input->param('basic_password') || undef;
64
my $basic_realm = $input->param('basic_realm') || undef;
65
my $datetime_granularity = $input->param('datetime_granularity');
66
my $record_type = $input->param('record_type');
67
my $original_system_field = $input->param('original_system_field') || undef;
68
69
if ( $op ){
70
    my $repository;
71
    if ( $repository_id ){
72
        #Fetch repository from database...
73
        $repository = Koha::OAI::Client::Repositories->find($repository_id);
74
    }
75
76
    if ( $op eq "new" ){
77
        $template->param(
78
            frameworkloop => C4::Koha::GetFrameworksLoop(),
79
        );
80
    } elsif ( $op eq "create" ){
81
        my $new_repository = Koha::OAI::Client::Repository->new({
82
            base_url => $base_url,
83
            metadata_prefix => $metadata_prefix,
84
            opt_from => $opt_from,
85
            opt_until => $opt_until,
86
            opt_set => $opt_set,
87
            active => $active,
88
            xslt_path => $xslt_path,
89
            frameworkcode => $frameworkcode,
90
            comments => $comments,
91
            basic_username => $basic_username,
92
            basic_password => $basic_password,
93
            basic_realm => $basic_realm,
94
            datetime_granularity => $datetime_granularity,
95
            record_type => $record_type,
96
            original_system_field => $original_system_field,
97
        });
98
        my $errors = $new_repository->validate();
99
        if (%$errors){
100
            $op = "new";
101
            $template->param(
102
                repository => $new_repository,
103
                frameworkloop => C4::Koha::GetFrameworksLoop($new_repository->frameworkcode),
104
                errors => $errors,
105
            );
106
        } else {
107
            $new_repository->store();
108
            $op = "list";
109
        }
110
    } elsif ( $op eq "edit" ){
111
        if ($repository){
112
            $template->param(
113
                repository => $repository,
114
                frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode),
115
            );
116
        }
117
    } elsif ( ($op eq "view") || ($op eq "reset") ){
118
        if ($repository){
119
120
            #If op eq reset, then delete the entire harvest for this repository
121
            if ($op eq "reset"){
122
                eval {
123
                    $repository->delete_harvest();
124
                    $template->param( reset_success => 1 );
125
                };
126
                if ($@){
127
                    my $problem_records = $@;
128
                    $template->param( problem_records => $problem_records );
129
                }
130
            }
131
132
            #"add", "update"
133
            my $imported_records = Koha::OAI::Client::Records->search({
134
                repository_id => $repository->repository_id,
135
                system_number => {'!=', undef},
136
            }, {
137
              group_by => [ qw/system_number/ ],
138
              #Group results by system_number so you only get 1 result if you have many rows sharing the same system_number
139
              #Also, group by system number because you might have imported record 1, deleted it, and then it gets re-imported later as a different Koha record
140
            });
141
            my $waiting_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_import',});
142
            my $to_delete_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'to_delete',});
143
            my $deleted_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'deleted',});
144
            my $error_records = Koha::OAI::Client::Records->search({ repository_id => $repository->repository_id, action => 'error',});
145
146
            $template->param(
147
                repository => $repository,
148
                imported_records_count => $imported_records->count(),
149
                waiting_records_count => $waiting_records->count(),
150
                to_delete_records_count => $to_delete_records->count(),
151
                deleted_records_count => $deleted_records->count(),
152
                error_records_count => $error_records->count(),
153
            );
154
        }
155
        if ($op eq "reset"){
156
            $op = "view";
157
        }
158
    } elsif ( $op eq "update" ){
159
        if ($repository){
160
            $repository->set({
161
                base_url => $base_url,
162
                metadata_prefix => $metadata_prefix,
163
                opt_from => $opt_from,
164
                opt_until => $opt_until,
165
                opt_set => $opt_set,
166
                active => $active,
167
                xslt_path => $xslt_path,
168
                frameworkcode => $frameworkcode,
169
                comments => $comments,
170
                basic_username => $basic_username,
171
                basic_password => $basic_password,
172
                basic_realm => $basic_realm,
173
                datetime_granularity => $datetime_granularity,
174
                record_type => $record_type,
175
                original_system_field => $original_system_field,
176
            });
177
            my $errors = $repository->validate();
178
            if (%$errors || $test_settings){
179
                if (! %$errors && $test_settings){
180
                    my $test_errors = $repository->test_settings();
181
                    if (%$test_errors){
182
                        $template->param( test_errors => $test_errors );
183
                    } else {
184
                        $template->param( test_success => 1 );
185
                    }
186
                }
187
                $op = "edit";
188
                $template->param(
189
                    repository => $repository,
190
                    frameworkloop => C4::Koha::GetFrameworksLoop($repository->frameworkcode),
191
                    errors => $errors,
192
                );
193
            } else {
194
                $repository->store();
195
                $op = "list";
196
            }
197
        }
198
    } elsif ( $op eq "delete" ){
199
        if ($repository){
200
            $repository->delete;
201
        }
202
        $op = "list";
203
    }
204
} else {
205
    $op = "list";
206
}
207
208
if ( $op eq "list" ){
209
        my @repositories = Koha::OAI::Client::Repositories->as_list();
210
        $template->param(
211
            repositories => \@repositories,
212
            frameworkloop => C4::Koha::GetFrameworksLoop(),
213
        );
214
}
215
216
$template->param(
217
    op => $op,
218
);
219
220
output_html_with_http_headers $input, $cookie, $template->output;
(-)a/installer/data/mysql/atomicupdate/bug_10662-Build_OAI-PMH_Harvesting_Client_tables.sql (+51 lines)
Line 0 Link Here
1
DROP TABLE IF EXISTS oai_harvest;
2
DROP TABLE IF EXISTS oai_harvest_repositories;
3
4
--
5
-- Table structure for table `oai_harvest_repositories`
6
--
7
8
CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
9
  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
10
  base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
11
  basic_username text, -- username for basic http authentication
12
  basic_password text, -- password for basic http authentication
13
  basic_realm text, -- realm for basic http authentication
14
  datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request
15
  opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob)
16
  opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional)
17
  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
18
  metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
19
  active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
20
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name.
21
  xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
22
  frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records
23
  comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
24
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
25
  original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c)
26
  -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number)
27
  PRIMARY KEY (repository_id)
28
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
29
30
--
31
-- Table structure for table `oai_harvest`
32
--
33
34
CREATE TABLE IF NOT EXISTS oai_harvest (
35
  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record
36
  repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested
37
  system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record
38
  action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore)
39
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name.
40
  identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository
41
  datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ).
42
  metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.)
43
  metadata longtext, -- original metadata as transmitted via OAI-PMH
44
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
45
  original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record)
46
  -- NOTE: In future, may need to add fields for holdings-biblio relationship
47
  PRIMARY KEY (oai_harvest_id),
48
  UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp),
49
  KEY FK_oai_harvest_1 (repository_id),
50
  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
51
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
(-)a/installer/data/mysql/kohastructure.sql (+49 lines)
Lines 3549-3554 CREATE TABLE discharges ( Link Here
3549
  CONSTRAINT borrower_discharges_ibfk1 FOREIGN KEY (borrower) REFERENCES borrowers (borrowernumber) ON DELETE CASCADE ON UPDATE CASCADE
3549
  CONSTRAINT borrower_discharges_ibfk1 FOREIGN KEY (borrower) REFERENCES borrowers (borrowernumber) ON DELETE CASCADE ON UPDATE CASCADE
3550
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3550
) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci;
3551
3551
3552
--
3553
-- Table structure for table `oai_harvest_repositories`
3554
--
3555
3556
CREATE TABLE IF NOT EXISTS oai_harvest_repositories (
3557
  repository_id int(11) NOT NULL AUTO_INCREMENT, -- primary key identifier
3558
  base_url text NOT NULL, -- baseURL of the remote OAI-PMH repository (mandatory)
3559
  basic_username text, -- username for basic http authentication
3560
  basic_password text, -- password for basic http authentication
3561
  basic_realm text, -- realm for basic http authentication
3562
  datetime_granularity enum('YYYY-MM-DDThh:mm:ssZ','YYYY-MM-DD') NOT NULL DEFAULT 'YYYY-MM-DD', -- granularity to use in harvest request
3563
  opt_from DATETIME DEFAULT NULL, -- "from" DATETIME for selective harvesting (optional - automatically updated by cronjob)
3564
  opt_until DATETIME DEFAULT NULL, -- "until" DATETIME for selective harvesting (optional)
3565
  opt_set varchar(45) DEFAULT NULL, -- the record set to harvest for selective harvesting (optional)
3566
  metadata_prefix varchar(45) NOT NULL, -- metadata format (e.g. oai_dc, dc, marcxml)
3567
  active int(1) NOT NULL DEFAULT '0', -- indicate whether this repo is actively harvested by Koha
3568
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- date last modified. Maybe `datelastmodified` would be a better column name.
3569
  xslt_path text, -- filepath to a local or external XSLT file to use for transforming incoming records into MARCXML
3570
  frameworkcode varchar(4) NOT NULL DEFAULT '', -- framework to use when ingesting records
3571
  comments text, -- limited number of characters (controlled by template) to describe the repository (optional - for librarian use rather than system use)
3572
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
3573
  original_system_field varchar(45), -- Field where you can a find a record's original system number (e.g. 001 or 999$c)
3574
  -- NOTE: In future, may need to add fields for holdings-biblio relationship (e.g. field to designate 004 in holdings record as link to a biblio's original system number)
3575
  PRIMARY KEY (repository_id)
3576
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
3577
3578
--
3579
-- Table structure for table `oai_harvest`
3580
--
3581
3582
CREATE TABLE IF NOT EXISTS oai_harvest (
3583
  oai_harvest_id int(11) NOT NULL AUTO_INCREMENT, -- unique internal id for the OAI record
3584
  repository_id int(11) NOT NULL, -- OAI repository from which the record was harvested
3585
  system_number int(11) DEFAULT NULL, -- indicates the Koha record created from this record
3586
  action varchar(45) DEFAULT NULL, -- indicates whether this record has been processed and how it's been processed (e.g. add, update, ignore)
3587
  timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, -- last modified date in Koha. Maybe `datelastmodified` would be a better column name.
3588
  identifier varchar(255) NOT NULL, -- the record's OAI ID from the OAI repository
3589
  datestamp varchar(45) NOT NULL, -- the record's OAI datestamp from the OAI repository (e.g. YYYY-MM-DDTHH:MM:SSZ).
3590
  metadata_prefix varchar(45) NOT NULL, -- the type of metadata (e.g. marcxml, oai_dc, etc.)
3591
  metadata longtext, -- original metadata as transmitted via OAI-PMH
3592
  record_type enum('biblio','auth','holdings') NOT NULL DEFAULT 'biblio',
3593
  original_system_number varchar(45), -- original system number (e.g. the unique number in the 001 of the incoming record)
3594
  -- NOTE: In future, may need to add fields for holdings-biblio relationship
3595
  PRIMARY KEY (oai_harvest_id),
3596
  UNIQUE KEY harvest_identifier_datestamp (identifier,datestamp),
3597
  KEY FK_oai_harvest_1 (repository_id),
3598
  CONSTRAINT FK_oai_harvest_1 FOREIGN KEY (repository_id) REFERENCES oai_harvest_repositories (repository_id) ON UPDATE NO ACTION
3599
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
3600
3552
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
3601
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
3553
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
3602
/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
3554
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
3603
/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
(-)a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc (+1 lines)
Lines 62-67 Link Here
62
<ul>
62
<ul>
63
	<!-- <li><a href="/cgi-bin/koha/admin/printers.pl">Network Printers</a></li> -->
63
	<!-- <li><a href="/cgi-bin/koha/admin/printers.pl">Network Printers</a></li> -->
64
    <li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>
64
    <li><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></li>
65
    <li><a href="/cgi-bin/koha/admin/oai_client.pl">OAI-PMH servers</a></li>
65
    <li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>
66
    <li><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></li>
66
    <li><a href="/cgi-bin/koha/admin/columns_settings.pl">Columns settings</a></li>
67
    <li><a href="/cgi-bin/koha/admin/columns_settings.pl">Columns settings</a></li>
67
</ul>
68
</ul>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt (+2 lines)
Lines 107-112 Link Here
107
	<dd>Printers (UNIX paths).</dd> -->
107
	<dd>Printers (UNIX paths).</dd> -->
108
    <dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>
108
    <dt><a href="/cgi-bin/koha/admin/z3950servers.pl">Z39.50/SRU servers</a></dt>
109
    <dd>Define which external servers to query for MARC data.</dd>
109
    <dd>Define which external servers to query for MARC data.</dd>
110
    <dt><a href="/cgi-bin/koha/admin/oai_client.pl">OAI-PMH servers</a></dt>
111
    <dd>Define external servers from which to harvest records via OAI-PMH.</dd>
110
    <dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>
112
    <dt><a href="/cgi-bin/koha/admin/didyoumean.pl">Did you mean?</a></dt>
111
    <dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>
113
    <dd>Choose which plugins to use to suggest searches to patrons and staff.</dd>
112
    <dt><a href="/cgi-bin/koha/admin/columns_settings.pl">Configure columns</a></dt>
114
    <dt><a href="/cgi-bin/koha/admin/columns_settings.pl">Configure columns</a></dt>
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_client.tt (+326 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Administration &rsaquo; OAI-PMH server targets</title>
3
[% INCLUDE 'doc-head-close.inc' %]
4
[% INCLUDE 'calendar.inc' %]
5
<script type="text/javascript" src="[% interface %]/lib/jquery/plugins/jquery-ui-timepicker-addon.min.js"></script>
6
[% INCLUDE 'timepicker.inc' %]
7
[% IF ( op == "list" ) %]
8
    <link rel="stylesheet" type="text/css" href="[% themelang %]/css/datatables.css" />
9
    [% INCLUDE 'datatables.inc' %]
10
    <script type="text/javascript">
11
    //<![CDATA[
12
        $(document).ready(function() {
13
            $("#serverst").dataTable($.extend(true, {}, dataTablesDefaults, {
14
                "aoColumnDefs": [
15
                    { "aTargets": [8,9], "bSortable": false },
16
                ],
17
                "sPaginationType": "four_button"
18
            }));
19
        });
20
    //]]>
21
    </script>
22
[% ELSIF ( op == "edit" ) || ( op == "new" ) %]
23
    <script type="text/javascript">
24
    //<![CDATA[
25
        $(document).ready(function() {
26
            $(".opt_datetime").datetimepicker({
27
                dateFormat: "yy-mm-dd",
28
                timeFormat: "HH:mm:ss",
29
                hour: 0,
30
                minute: 0,
31
                second: 0,
32
                showSecond: 1,
33
            });
34
        });
35
    //]]>
36
    </script>
37
    <style type="text/css">
38
        /* Override staff-global.css which hides second, millisecond, and microsecond sliders */
39
        .ui_tpicker_second {
40
            display: block;
41
        }
42
        .test-success {
43
            /* same color as .text-success in Bootstrap 2.2.2 */
44
            color:#468847;
45
        }
46
    </style>
47
[% END %]
48
</head>
49
50
<body id="admin_oai_server_targets" class="admin">
51
[% INCLUDE 'header.inc' %]
52
[% INCLUDE 'cat-search.inc' %]
53
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/admin/admin-home.pl">Administration</a> &rsaquo; OAI-PMH server targets</div>
54
55
<div id="doc3" class="yui-t2">
56
57
<div id="bd">
58
  <div id="yui-main">
59
    <div class="yui-b">
60
        [% IF ( op ) %]
61
            [% IF ( op == "list" ) %]
62
                <div id="toolbar" class="btn-toolbar">
63
                    <a id="newserver" class="btn btn-small" href="/cgi-bin/koha/admin/oai_client.pl?op=new"><i class="icon-plus"></i> New OAI-PMH server target</a>
64
                </div>
65
                <h1>OAI-PMH server targets</h1>
66
                <table id="serverst">
67
                    <thead>
68
                        <tr>
69
                            <th>Base URL</th>
70
                            <th>Metadata Prefix</th>
71
                            <th>From</th>
72
                            <th>Until</th>
73
                            <th>Set</th>
74
                            <th>Active</th>
75
                            <th>Path to XSLT</th>
76
                            <th>Record type</th>
77
                            <th>MARC framework</th>
78
                            <th>Original system field</th>
79
                            <th>Comments</th>
80
                            <th>Options</th>
81
                        </tr>
82
                    </thead>
83
                    <tbody>
84
                    [% FOREACH repository IN repositories %]
85
                        <tr>
86
                            <td>[% repository.base_url %]</td>
87
                            <td>[% repository.metadata_prefix %]</td>
88
                            <td>[% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %]</td>
89
                            <td>[% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %]</td>
90
                            <td>[% repository.opt_set %]</td>
91
                            <td>
92
                                [% IF ( repository.active == 1 ) %]
93
                                    Active
94
                                [% ELSE %]
95
                                    Inactive
96
                                [% END %]
97
                            </td>
98
                            <td>[% repository.xslt_path %]</td>
99
                            <td>[% repository.record_type %]</td>
100
                            <td>
101
                                [%
102
                                    framework_name = '';
103
                                    FOREACH framework IN frameworkloop;
104
                                        IF ( framework.value == repository.frameworkcode );
105
                                            framework_name = framework.description;
106
                                        END;
107
                                    END;
108
                                %]
109
                                [% IF ( framework_name ) %]
110
                                    [% framework_name %]
111
                                [% ELSE %]
112
                                    Default
113
                                [% END %]
114
                            </td>
115
                            <td>[% repository.original_system_field %]</td>
116
                            <td>[% repository.comments %]</td>
117
                            <td>
118
                                <a href="/cgi-bin/koha/admin/oai_client.pl?op=view&repository_id=[% repository.repository_id %]">View</a>
119
                                <a href="/cgi-bin/koha/admin/oai_client.pl?op=edit&repository_id=[% repository.repository_id %]">Edit</a>
120
                                <a href="/cgi-bin/koha/admin/oai_client.pl?op=delete&repository_id=[% repository.repository_id %]">Delete</a>
121
                            </td>
122
                        </tr>
123
                    [% END %]
124
                    </tbody>
125
                </table>
126
            [% ELSIF ( op == "view" ) %]
127
                <h1>View OAI-PMH server target</h1>
128
                <h2>[% repository.base_url %]</h2>
129
                <fieldset class="rows">
130
                    <ol>
131
                        <li>Koha records created from harvested records: [% imported_records_count %]</li>
132
                        <li>Koha records deleted by harvested records: [% deleted_records_count %]</li>
133
                        <li>Harvested records waiting to be imported: [% waiting_records_count %]</li>
134
                        <li>Harvested records waiting to be deleted from Koha: [% to_delete_records_count %]</li>
135
                        <li>Harvested records in an error state: [% error_records_count %]</li>
136
                    </ol>
137
                </fieldset>
138
                <form action="/cgi-bin/koha/admin/oai_client.pl" name="reset-form" method="get" id="oai-server-reset" novalidate="novalidate">
139
                    <fieldset class="action">
140
                        <input type="hidden" name="op" value="reset" />
141
                        <input type="hidden" name="repository_id" value="[% repository.repository_id %]" />
142
                        <button type="submit">Reset repository harvest</button>
143
                    </fieldset>
144
                </form>
145
                [% IF ( problem_records ) %]
146
                    <p>Not all of this repository's records could be deleted. Manual intervention may be required to remove items. Review the following records:</p>
147
                    <ol>
148
                    [% FOREACH problem_record IN problem_records %]
149
                        [% IF ( problem_record.record_type == "biblio" ) %]
150
                        <li><a target="_blank" href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% problem_record.system_number %]">[% problem_record.record_type %] - [% problem_record.system_number %]</a></li>
151
                        [% ELSE %]
152
                        <li>[% problem_record.record_type %] - [% problem_record.system_number %]</li>
153
                        [% END %]
154
                    [% END %]
155
                    </ol>
156
                [% END %]
157
                [% IF ( reset_success ) %]
158
                    <p>This repository no longer has any harvested records. Modify the "From" and "Until" dates to prepare the repository for the next harvest.</p>
159
                [% END %]
160
            [% ELSIF ( op == "edit" ) || ( op == "new" ) %]
161
                [% IF ( op == "new" ) %]
162
                    <h1>New OAI-PMH server target</h1>
163
                [% ELSIF ( op == "edit" ) %]
164
                    <h1>Modify OAI-PMH server target</h1>
165
                [% END %]
166
                <form action="/cgi-bin/koha/admin/oai_client.pl" name="detail-form" method="post" id="oai-server-target-details" novalidate="novalidate">
167
                    [% IF ( op == "new" ) %]
168
                        <input type="hidden" name="op" value="create" />
169
                    [% ELSIF ( op == "edit" ) %]
170
                        <input type="hidden" name="repository_id" value="[% repository.repository_id %]" />
171
                        <input type="hidden" name="op" value="update" />
172
                    [% END %]
173
                    <fieldset class="rows">
174
                        <legend>HTTP parameters:</legend>
175
                        <ol>
176
                            <li>
177
                                <label for="base_url">Base URL: </label>
178
                                <input type="text" id="base_url" name="base_url" value="[% repository.base_url %]" size="60" />
179
                                [% IF ( errors.base_url.required ) %]<span class="required">Required</span>[% END %]
180
                                <span class="error">[% IF ( errors.base_url.invalid ) %](Invalid URL)[% END %]</span>
181
                                <span class="error">[% IF ( test_errors.url_not_found ) %](404 - URL Not Found)[% END %]</span>
182
183
                            </li>
184
                            <li>
185
                                <label for="basic_username">Basic auth username: </label>
186
                                <input type="text" id="basic_username" name="basic_username" value="[% repository.basic_username %]" size="60" />
187
                                <span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
188
                            </li>
189
                            <li>
190
                                <label for="basic_password">Basic auth password: </label>
191
                                <input type="text" id="basic_password" name="basic_password" value="[% repository.basic_password %]" size="60" />
192
                                <span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
193
                            </li>
194
                            <li>
195
                                <label for="basic_realm">Basic auth realm: </label>
196
                                <input type="text" id="basic_realm" name="basic_realm" value="[% repository.basic_realm %]" size="60" />
197
                                <span class="error">[% IF ( test_errors.failed_authentication ) %](Authentication failed)[% END %]</span>
198
                            </li>
199
                        </ol>
200
                    </fieldset>
201
                    <fieldset class="rows">
202
                        <legend>OAI-PMH parameters:</legend>
203
                        <ol>
204
                            <li>
205
                                <label for="opt_set">Set: </label>
206
                                <input type="text" name="opt_set" value="[% repository.opt_set %]" size="30" />
207
                                <span class="error">[% IF ( test_errors.set_does_not_exist ) %](Set unavailable)[% END %]</span>
208
                            </li>
209
                            <li>
210
                                <label for="metadata_prefix">Metadata prefix: </label>
211
                                <input type="text" name="metadata_prefix" value="[% repository.metadata_prefix %]" size="30" />
212
                                [% IF ( errors.metadata_prefix.required ) %]<span class="required">Required</span>[% END %]
213
                                [% IF ( test_errors.metadata_prefix_does_not_exist ) %]<span class="error">(Metadata prefix unavailable)</span>[% END %]
214
215
                            </li>
216
                            <li>
217
                                <label for="datetime_granularity">Granularity: </label>
218
                                <select name="datetime_granularity">
219
                                    [% FOREACH granularity IN ['YYYY-MM-DD', 'YYYY-MM-DDThh:mm:ssZ'] %]
220
                                        [% IF ( granularity == repository.datetime_granularity ) %]
221
                                        <option value="[% granularity %]" selected="selected">[% granularity %]</option>
222
                                        [% ELSE %]
223
                                        <option value="[% granularity %]">[% granularity %]</option>
224
                                        [% END %]
225
                                    [% END %]
226
                                </select>
227
                                <!-- <span class="required">Required</span> -->
228
                                [% IF ( test_errors.second_granularity_not_supported ) %]<span class="error">(Second granularity is not supported)</span>[% END %]
229
                            </li>
230
                            <li>
231
                                <label for="opt_from">From: </label>
232
                                <input type="text" class="opt_datetime" name="opt_from" value="[% IF ( repository.opt_from ) != "0000-00-00 00:00:00"; repository.opt_from; END; %]" size="30" />
233
                            </li>
234
                            <li>
235
                                <label for="opt_until">Until: </label>
236
                                <input type="text" class="opt_datetime" name="opt_until" value="[% IF ( repository.opt_until ) != "0000-00-00 00:00:00"; repository.opt_until; END; %]" size="30" />
237
                            </li>
238
                        </ol>
239
                    </fieldset>
240
                    <fieldset class="action">
241
                       [% IF ( test_success ) %]
242
                            <p class="test-success">HTTP and OAI-PMH parameter tests completed successfully.</p>
243
                       [% END %]
244
                       [% IF (test_errors.no_harvester) %]
245
                            <p>Unable to create a OAI-PMH harvester with these settings</p>
246
                       [% END %]
247
                       [% IF (test_errors.generic_identify_error) %]
248
                            <p>An error was encountered using the "Identify" verb with the server target.</p>
249
                       [% END %]
250
                        <input type="submit" name="test_settings" value="Test HTTP and OAI-PMH parameters">
251
                    </fieldset>
252
                    <fieldset class="rows">
253
                        <legend>Import parameters:</legend>
254
                        <ol>
255
                            <li>
256
                                <label for="active">Active: </label>
257
                                [% IF ( repository.active == 1) %]
258
                                    <input type="radio" name="active" value="1" checked="checked" />
259
                                [% ELSE %]
260
                                    <input type="radio" name="active" value="1" />
261
                                [% END %]
262
                                 Active
263
                                [% IF ( repository.active != 1) %]
264
                                    <input type="radio" name="active" value="0" checked="checked" />
265
                                [% ELSE %]
266
                                    <input type="radio" name="active" value="0" />
267
                                [% END %]
268
                                 Inactive
269
                            </li>
270
                            <li>
271
                                <label for="xslt_path">Path to XSLT: </label>
272
                                <input type="text" name="xslt_path" value="[% repository.xslt_path %]" size="60" />
273
                            </li>
274
                            <li>
275
                                <label for="record_type">Record type:</label>
276
                                <select name="record_type">
277
                                    [% record_types = [ 'biblio' ] #TODO: In future, add 'auth' and 'holdings' support here %]
278
                                    [% FOREACH type IN record_types %]
279
                                        [% IF type == repository.record_type %]
280
                                            <option value="[% type %]" selected="selected">[% type %]</option>
281
                                        [% ELSE %]
282
                                            <option value="[% type %]">[% type %]</option>
283
                                        [% END %]
284
                                    [% END %]
285
                                </select>
286
                                <!-- <span class="required">Required</span> -->
287
                            </li>
288
                            [%# Only need framework for 'biblio' and 'holdings'. This will need to change for 'auth'. %]
289
                            <li>
290
                                <label for="frameworkcode">MARC framework: </label>
291
                                <select name="frameworkcode">
292
                                    <option value="">Default</option>
293
                                [% FOREACH framework IN frameworkloop %]
294
                                    [% IF ( framework.selected ) %]
295
                                        <option value="[% framework.value %]" selected="selected">[% framework.description %]</option>
296
                                    [% ELSE %]
297
                                        <option value="[% framework.value %]">[% framework.description %]</option>
298
                                    [% END %]
299
                                [% END %]
300
                                </select>
301
                                <!-- <span class="required">Required</span> -->
302
                            </li>
303
                            <li>
304
                                <label for="original_system_field">Original system field</label>
305
                                <input type="text" name="original_system_field" value="[% repository.original_system_field %]" size="60%" />
306
                            </li>
307
                            <li>
308
                                <label for="comments">Comments: </label>
309
                                <textarea name="comments">[% repository.comments %]</textarea>
310
                            </li>
311
                        </ol>
312
                    </fieldset>
313
                    <fieldset class="action">
314
                        <input type="submit" value="Save">
315
                        <a class="cancel" href="/cgi-bin/koha/admin/oai_client.pl">Cancel</a>
316
                    </fieldset>
317
                </form>
318
            [% END %]
319
        [% END #/op %]
320
    </div>
321
  </div>
322
  <div class="yui-b">
323
    [% INCLUDE 'admin-menu.inc' %]
324
  </div>
325
</div>
326
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/MARC21slimFromOAI.xsl (+82 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8"?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:marc="http://www.loc.gov/MARC21/slim"
4
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
5
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
6
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
7
8
    <!-- pass in the OAI-PMH identifier for archival purposes -->
9
    <xsl:param name="identifier"/>
10
11
    <!-- This is the first template which matches the incoming metadata -->
12
    <!-- We strip out the OAI metadata wrapper by doing nothing but applying the templates for marc:record -->
13
    <xsl:template match="oai:metadata">
14
        <xsl:apply-templates />
15
    </xsl:template>
16
17
    <xsl:template match="marc:record">
18
        <xsl:copy>
19
            <!-- Apply all relevant templates for all attributes and elements -->
20
            <xsl:apply-templates select="@* | *" mode="copy"/>
21
22
            <!-- Add new node (or whatever else you want to do after copying the existing record) -->
23
            <xsl:if test="$identifier">
24
                <xsl:element name="datafield" xmlns="http://www.loc.gov/MARC21/slim">
25
                    <xsl:attribute name="ind1"><xsl:text>8</xsl:text></xsl:attribute>
26
                    <xsl:attribute name="ind2"><xsl:text> </xsl:text></xsl:attribute>
27
                    <xsl:attribute name="tag">037</xsl:attribute>
28
29
                    <xsl:element name="subfield">
30
                        <xsl:attribute name="code">a</xsl:attribute>
31
                        <xsl:value-of select="$identifier"/>
32
                    </xsl:element>
33
34
                    <xsl:element name="subfield">
35
                        <xsl:attribute name="code">b</xsl:attribute>
36
                        <xsl:text>OAI-PMH</xsl:text>
37
                    </xsl:element>
38
                </xsl:element>
39
            </xsl:if>
40
        </xsl:copy>
41
    </xsl:template>
42
43
44
    <!-- Identity transformation: this template is the workhorse that copies attributes and nodes -->
45
    <!-- In terms of nodes, it'll apply to the leader, controlfield, and subfields. It won't apply to datafields, as we have a more specific template for those. -->
46
    <xsl:template match="@* | node()" mode="copy">
47
        <!-- Create a copy of this attribute or node -->
48
        <xsl:copy>
49
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
50
            <xsl:apply-templates select="@* | node()" mode="copy"/>
51
        </xsl:copy>
52
    </xsl:template>
53
54
     <xsl:template match="marc:datafield" mode="copy">
55
        <!-- Add subfields by changing the predicate in the select attribute (e.g. [@code='9' or @code='a']) -->
56
        <xsl:variable name="child_subfields_to_remove" select="child::*[@code='9']"/>
57
        <!-- Strip out all $9 subfields, as these provide links to authority records. These will nearly never be correct linkages, so strip them out. -->
58
        <xsl:choose>
59
            <xsl:when test="self::node()[@tag = '952' or @tag = '942' or @tag = '999']">
60
            <!-- STRIP DATAFIELDS -->
61
            <!-- Add datafields to strip by changing the predicate in the test attribute -->
62
            <!-- Strip out any 952 tags so that we don't have indexing problems in regards to unexpected items... -->
63
            <!-- Strip out any 942 tags. They'll contain local data (e.g. 942$c item type) which is local and thus won't correspond with the Koha that is importing these records -->
64
            <!-- Strip all 999 fields; this isn't strictly necessary though, as "C4::Biblio::_koha_add_biblio" and "C4::Biblio::_koha_add_biblioitem" will typically fix the 999$c and 999$d fields in MARC21 -->
65
            <!-- NOTE: If you don't strip the 942 field, you'll need to make sure that the item type specified matches one that already exists in Koha or you'll have problems -->
66
            </xsl:when>
67
            <xsl:when test="count(child::*) = count($child_subfields_to_remove)">
68
            <!-- STRIP DATAFIELDS WHICH WILL BECOME EMPTY -->
69
            <!-- We don't want to output a datafield if we're going to remove all its children, as empty datafields cause fatal errors in MARC::Record -->
70
            </xsl:when>
71
            <xsl:otherwise>
72
                <!-- Create a copy of the datafield element (without attributes or child nodes) -->
73
                <xsl:copy>
74
                    <!-- Apply copy templates for datafield attributes -->
75
                    <xsl:apply-templates select="@*" mode="copy"/>
76
                    <!-- Apply copy templates for subfield nodes -->
77
                    <xsl:apply-templates select="marc:subfield[not(self::node() = $child_subfields_to_remove)]" mode="copy"/>
78
                </xsl:copy>
79
            </xsl:otherwise>
80
        </xsl:choose>
81
    </xsl:template>
82
</xsl:stylesheet>
(-)a/koha-tmpl/intranet-tmpl/prog/en/xslt/StripOAIWrapper.xsl (+22 lines)
Line 0 Link Here
1
<?xml version="1.0" encoding="UTF-8" ?>
2
<xsl:stylesheet version="1.0"
3
    xmlns:oai="http://www.openarchives.org/OAI/2.0/"
4
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
5
    <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
6
7
    <!-- This is the first template that matches the OAI-PMH root metadata wrapper -->
8
    <!-- We strip out the OAI-PMH metadata wrapper by doing nothing but applying the remaining templates to its child nodes -->
9
    <!-- NOTE: We need to strip the wrapper, so that we can access the actual record using modules like MARC::Record-->
10
    <xsl:template match="oai:metadata">
11
        <xsl:apply-templates />
12
    </xsl:template>
13
14
    <!-- Identity transformation: this template copies attributes and nodes -->
15
    <xsl:template match="@* | node()">
16
        <!-- Create a copy of this attribute or node -->
17
        <xsl:copy>
18
            <!-- Recursively apply this template to the attributes and child nodes of this element -->
19
            <xsl:apply-templates select="@* | node()" />
20
        </xsl:copy>
21
    </xsl:template>
22
</xsl:stylesheet>
(-)a/misc/cronjobs/oai/oai_harvester.pl (-1 / +180 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright Prosentient Systems 2015
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, see <http://www.gnu.org/licenses>.
18
19
use Modern::Perl;
20
use Getopt::Long;
21
use Pod::Usage;
22
23
24
=head1 NAME
25
26
oai_harvester.pl - cron script to download records via OAI-PMH, and import
27
into Koha.
28
29
=head1 SYNOPSIS
30
31
perl oai_harvester.pl [ -r ][ -d ][ -i ]
32
33
=head1 OPTIONS
34
35
=over
36
37
=item B<--help>
38
39
Print a brief help message and exits.
40
41
=item B<--man>
42
43
Prints the manual page and exits.
44
45
=item B<-v>
46
47
Verbose. Without this flag set, only errors are reported.
48
49
=item B<-r>
50
51
Re-harvest the OAI server by resetting the "from" parameter to null.
52
53
=item B<-d>
54
55
Download records from the OAI-PMH server, and queue for importing.
56
57
=item B<-i>
58
59
Import queued records into Koha.
60
61
=back
62
63
=head1 DESCRIPTION
64
65
With the "-d" option, this script iterates through a list of
66
OAI servers (ie repositories), issues a ListRecords OAI-PMH request, and
67
downloads the results into the Koha database to await further
68
processing (ie adding new records and updating existing ones).
69
70
With the "-i" option, this script iterates through a list of
71
harvested OAI-PMH metadata records. Each record is checked
72
to see whether it needs to be added as a new record in Koha,
73
or if it needs to update a record that has previously been
74
harvested. After this checking, it is optionally transformed
75
using a user-defined XSLT. Finally, it is passed to
76
Koha's internal APIs to be added/updated in the database.
77
78
79
=cut
80
81
my $help    = 0;
82
my $man     = 0;
83
my $verbose = 0;
84
my $reset = 0;
85
my $download = 0;
86
my $import = 0;
87
88
GetOptions(
89
            'h|help|?'          => \$help,
90
            'm|man'             => \$man,
91
            'v'                 => \$verbose,
92
            'd'                 => \$download,
93
            'i'                 => \$import,
94
            'r'                 => \$reset,
95
       )or pod2usage(2);
96
pod2usage(1) if $help;
97
pod2usage( -verbose => 2 ) if $man;
98
99
if ($download){
100
    require C4::Context;
101
    require XML::LibXSLT;
102
    require Koha::OAI::Client::Repositories;
103
104
    my $stylesheet;
105
    #Build path to XSLT filename
106
    #FIXME: It would make more sense to have a utility XSLT directory outside of htdocs which is language independent...
107
    my $htdocs  = C4::Context->config('intrahtdocs');
108
    my $theme   = C4::Context->preference("template");
109
    my $xslfilename = "$htdocs/$theme/en/xslt/StripOAIWrapper.xsl";
110
    if ( -f $xslfilename ){
111
        #FIXME: Ideally, it would be good to use Koha::XSLT_Handler here, but
112
        #we want to output a XML::LibXML::Document instead of output_as_chars.
113
        my $xslt = XML::LibXSLT->new();
114
        my $style_doc = XML::LibXML->load_xml(location => $xslfilename);
115
        $stylesheet = $xslt->parse_stylesheet($style_doc);
116
    }
117
118
    my $repositories = Koha::OAI::Client::Repositories->search({
119
        active => 1,
120
        base_url => {'!=', undef},
121
    });
122
    while (my $repository = $repositories->next){
123
        if ($reset){
124
            $verbose && print "Resetting opt_from to undefined...\n";
125
            $repository->opt_from(undef);
126
        }
127
        $repository->harvest({ verbose => $verbose, stylesheet => $stylesheet, });
128
    }
129
}
130
131
if ($import){
132
    require Koha::OAI::Client::Importer;
133
    require Koha::OAI::Client::Records;
134
135
    #Import records, previously harvested via OAI-PMH, into Koha
136
    my $oai_importer = Koha::OAI::Client::Importer->new();
137
    if ($oai_importer){
138
        my $records = Koha::OAI::Client::Records->search({
139
            system_number => undef,
140
            action => ["to_import","to_delete"],
141
        });
142
143
        $verbose && print $records->count." records to process\n";
144
        if ($records){
145
146
            my $number_imported = 0;
147
            my $number_deleted = 0;
148
149
            #Iterate through the records and import (i.e. add/update/re-add) or delete them in Koha
150
            while( my $record = $records->next ) {
151
                if ($record->action eq "to_import"){
152
                    eval {
153
                        $oai_importer->import_record($record);
154
                    };
155
                    if ($@){
156
                        warn "Error importing record (oai_harvest_id ".$record->oai_harvest_id.") into Koha:\n$@";
157
                        $record->action("error");
158
                        $record->store();
159
                    } else {
160
                        $verbose && print $record->action . " identifier (".$record->identifier.") (".$record->record_type.") as system number (".$record->system_number.")\n";
161
                        $number_imported++;
162
                    }
163
                } elsif ($record->action eq "to_delete") {
164
                    eval {
165
                        $oai_importer->delete_record($record);
166
                    };
167
                    if ($@){
168
                        warn "Error deleting record (oai_harvest_id ".$record->oai_harvest_id.") from Koha:\n$@";
169
                    } else {
170
                        $number_deleted++;
171
                    }
172
                }
173
            }
174
            $verbose && print "$number_imported records imported (i.e. added/updated)\n";
175
            $verbose && print "$number_deleted records deleted \n";
176
177
178
        }
179
    }
180
}

Return to bug 10662