View | Details | Raw Unified | Return to bug 35659
Collapse All | Expand All

(-)a/Koha/OAI/Client/Harvester.pm (-4 / +9 lines)
Lines 87-96 Starts harvesting Link Here
87
87
88
sub init {
88
sub init {
89
    my ( $self, $args ) = @_;
89
    my ( $self, $args ) = @_;
90
91
    $self->printlog("Starting OAI Harvest from repository");
90
    $self->printlog("Starting OAI Harvest from repository");
92
    my $server = $self->{server};
91
    my $server = $self->{server};
93
    my $days   = $self->{days};
92
    my $days   = $self->{days};
93
    my $init_results;
94
94
95
    my $h = HTTP::OAI::Harvester->new(
95
    my $h = HTTP::OAI::Harvester->new(
96
        repository => HTTP::OAI::Identify->new(
96
        repository => HTTP::OAI::Identify->new(
Lines 117-122 sub init { Link Here
117
            $lmdflog .= $_->metadataPrefix . " ";
117
            $lmdflog .= $_->metadataPrefix . " ";
118
        }
118
        }
119
        $self->printlog("Available metadata formats for this repository: $lmdflog");
119
        $self->printlog("Available metadata formats for this repository: $lmdflog");
120
        $init_results->{metadata_formats} = $lmdflog;
120
    } catch {
121
    } catch {
121
        $self->printlog("ListMetadataFormats failed");
122
        $self->printlog("ListMetadataFormats failed");
122
    };
123
    };
Lines 148-154 sub init { Link Here
148
    );
149
    );
149
    if ( $response->is_error ) {
150
    if ( $response->is_error ) {
150
        $self->printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
151
        $self->printlog( "Error requesting ListRecords: " . $response->code . " " . $response->message );
151
        exit;
152
        $init_results->{is_error} = $response->message;
153
        return $init_results;
152
    }
154
    }
153
    $self->printlog( "Request URL: " . $response->requestURL );
155
    $self->printlog( "Request URL: " . $response->requestURL );
154
156
Lines 164-169 sub init { Link Here
164
        my $status = $self->processRecord($oai_record);
166
        my $status = $self->processRecord($oai_record);
165
        $stats{$status}++;
167
        $stats{$status}++;
166
    }
168
    }
169
    $init_results->{total} = $stats{'total'};
167
170
168
    my $results = '';
171
    my $results = '';
169
    foreach my $status (@statuses) {
172
    foreach my $status (@statuses) {
Lines 193-212 sub init { Link Here
193
            )
196
            )
194
        ) or $self->printlog("OAI_HARVEST_REPORT letter not found");
197
        ) or $self->printlog("OAI_HARVEST_REPORT letter not found");
195
198
196
        my $success = C4::Letters::EnqueueLetter(
199
        my $message_id = C4::Letters::EnqueueLetter(
197
            {
200
            {
198
                letter     => $letter,
201
                letter     => $letter,
199
                to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email'
202
                to_address => C4::Context->preference("OAI-PMH:HarvestEmailReport"), message_transport_type => 'email'
200
            }
203
            }
201
        );
204
        );
202
        if ($success) {
205
        if ($message_id) {
203
            $self->printlog("Email report enqueued");
206
            $self->printlog("Email report enqueued");
207
            $init_results->{letter_message_id} = $message_id;
204
        } else {
208
        } else {
205
            $self->printlog("Unable to enqueue report email");
209
            $self->printlog("Unable to enqueue report email");
206
        }
210
        }
207
    }
211
    }
208
212
209
    $self->printlog("Ending OAI Harvest from repository");
213
    $self->printlog("Ending OAI Harvest from repository");
214
    return $init_results;
210
}
215
}
211
216
212
=head2 processRecord
217
=head2 processRecord
(-)a/t/db_dependent/Koha/OAIHarvester.t (-31 / +62 lines)
Lines 18-28 Link Here
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
18
# along with Koha; if not, see <http://www.gnu.org/licenses>.
19
19
20
use Modern::Perl;
20
use Modern::Perl;
21
use Test::More tests => 24;
21
use Test::More tests => 28;
22
use Test::Exception;
22
use Test::Exception;
23
use File::Temp qw/tempfile/;
23
use File::Temp qw/tempfile/;
24
24
25
use t::lib::TestBuilder;
25
use t::lib::TestBuilder;
26
use t::lib::Mocks;
27
26
use HTTP::OAI;
28
use HTTP::OAI;
27
use HTTP::OAI::Metadata::OAI_DC;
29
use HTTP::OAI::Metadata::OAI_DC;
28
use HTTP::OAI::Record;
30
use HTTP::OAI::Record;
Lines 39-46 $schema->storage->txn_begin; Link Here
39
41
40
my $new_oai_biblio = Koha::OAIServer->new(
42
my $new_oai_biblio = Koha::OAIServer->new(
41
    {
43
    {
42
        endpoint   => 'my_host1.org',
44
        endpoint => C4::Context->preference('OPACBaseURL')
43
        oai_set    => 'set1',
45
          . '/cgi-bin/koha/oai.pl',
46
        oai_set    => '',
44
        servername => 'my_test_1',
47
        servername => 'my_test_1',
45
        dataformat => 'oai_dc',
48
        dataformat => 'oai_dc',
46
        recordtype => 'biblio',
49
        recordtype => 'biblio',
Lines 48-57 my $new_oai_biblio = Koha::OAIServer->new( Link Here
48
    }
51
    }
49
)->store;
52
)->store;
50
53
51
my $harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_biblio, verbose => 1, days => 1, force => 1 } );
54
C4::Context->set_preference( 'OAI-PMH', 1 );
55
t::lib::Mocks::mock_preference( 'OAI-PMH:HarvestEmailReport',
56
    C4::Context->preference('KohaAdminEmailAddress') );
57
58
my $harvester = Koha::OAI::Client::Harvester->new(
59
    { server => $new_oai_biblio, verbose => 1, days => 1, force => 1 } );
60
61
my $init_results = $harvester->init();
62
63
like(
64
    $init_results->{metadata_formats},
65
    qr/oai_dc marc21 marcxml/,
66
    'Got list of supported metadata formats'
67
);
68
is( $init_results->{is_error}, undef, 'ListRecords request worked' );
69
cmp_ok( $init_results->{total}, '>', 0, 'Records have been processed' );
70
isnt( $init_results->{letter_message_id}, undef, 'Report has been enqueued' );
52
71
53
my $record =
72
my $record =
54
    '<metadata xmlns="http://www.openarchives.org/OAI/2.0/"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:title>Pièces diverses </dc:title><dc:identifier>ARCH/0320 [cote]</dc:identifier><dc:relation>FR-920509801 [RCR établissement]</dc:relation><dc:relation>Central obrera boliviana [Fonds ou collection]</dc:relation><dc:format>1 carton</dc:format><dc:date>1971/2000</dc:date><dc:type>Archives et manuscrits</dc:type></oai_dc:dc></metadata>';
73
'<metadata xmlns="http://www.openarchives.org/OAI/2.0/"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:title>Pièces diverses </dc:title><dc:identifier>ARCH/0320 [cote]</dc:identifier><dc:relation>FR-920509801 [RCR établissement]</dc:relation><dc:relation>Central obrera boliviana [Fonds ou collection]</dc:relation><dc:format>1 carton</dc:format><dc:date>1971/2000</dc:date><dc:type>Archives et manuscrits</dc:type></oai_dc:dc></metadata>';
55
74
56
my $r = HTTP::OAI::Record->new();
75
my $r = HTTP::OAI::Record->new();
57
$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) );
76
$r->metadata( HTTP::OAI::Metadata->new( dom => $record ) );
Lines 78-92 is( $status, 'added', 'Record is added' ); Link Here
78
$status = $harvester->processRecord($r);
97
$status = $harvester->processRecord($r);
79
is( $status, 'updated', 'When force is used, record is updated' );
98
is( $status, 'updated', 'When force is used, record is updated' );
80
99
81
$harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_biblio, verbose => 1, days => 1, force => 0 } );
100
$harvester = Koha::OAI::Client::Harvester->new(
82
$status    = $harvester->processRecord($r);
101
    { server => $new_oai_biblio, verbose => 1, days => 1, force => 0 } );
83
is( $status, 'skipped', 'When force is not used, record is skipped (already up to date)' );
102
$status = $harvester->processRecord($r);
103
is( $status, 'skipped',
104
    'When force is not used, record is skipped (already up to date)' );
84
105
85
$r->header->datestamp('2018-05-10T09:18:13Z');
106
$r->header->datestamp('2018-05-10T09:18:13Z');
86
$status = $harvester->processRecord($r);
107
$status = $harvester->processRecord($r);
87
is( $status, 'updated', 'When force is not used, record is updated if newer' );
108
is( $status, 'updated', 'When force is not used, record is updated if newer' );
88
109
89
my $imported_record = Koha::Import::OAI::Biblios->find( { identifier => 'oai:myarchive.org:oid-234' } );
110
my $imported_record = Koha::Import::OAI::Biblios->find(
111
    { identifier => 'oai:myarchive.org:oid-234' } );
90
my $added_datestamp = '2017-05-10T09:18:13Z';
112
my $added_datestamp = '2017-05-10T09:18:13Z';
91
$imported_record->update(
113
$imported_record->update(
92
    {
114
    {
Lines 96-113 $imported_record->update( Link Here
96
118
97
$r->header->datestamp(undef);
119
$r->header->datestamp(undef);
98
$status          = $harvester->processRecord($r);
120
$status          = $harvester->processRecord($r);
99
$imported_record = Koha::Import::OAI::Biblios->find( { identifier => 'oai:myarchive.org:oid-234' } );
121
$imported_record = Koha::Import::OAI::Biblios->find(
122
    { identifier => 'oai:myarchive.org:oid-234' } );
100
my $updated_datestamp = $imported_record->datestamp;
123
my $updated_datestamp = $imported_record->datestamp;
101
isnt(
124
isnt( $added_datestamp, $updated_datestamp,
102
    $added_datestamp, $updated_datestamp,
125
'local datestamp is updated even if there is no datestamp in incoming record'
103
    'local datestamp is updated even if there is no datestamp in incoming record'
104
);
126
);
105
127
106
$r->header->status('deleted');
128
$r->header->status('deleted');
107
$status = $harvester->processRecord($r);
129
$status = $harvester->processRecord($r);
108
is( $status, 'deleted', 'When a record is marked to be deleted, status is deleted' );
130
is( $status, 'deleted',
131
    'When a record is marked to be deleted, status is deleted' );
109
132
110
$imported_record = Koha::Import::OAI::Biblios->find( { identifier => 'oai:myarchive.org:oid-234' } );
133
$imported_record = Koha::Import::OAI::Biblios->find(
134
    { identifier => 'oai:myarchive.org:oid-234' } );
111
is( $imported_record, undef, 'Record has been deleted' );
135
is( $imported_record, undef, 'Record has been deleted' );
112
136
113
$status = $harvester->processRecord($r);
137
$status = $harvester->processRecord($r);
Lines 126-135 my $new_oai_auth = Koha::OAIServer->new( Link Here
126
    }
150
    }
127
)->store;
151
)->store;
128
152
129
$harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_auth, verbose => 1, days => 1, force => 1 } );
153
$harvester = Koha::OAI::Client::Harvester->new(
154
    { server => $new_oai_auth, verbose => 1, days => 1, force => 1 } );
130
155
131
my $auth =
156
my $auth =
132
    '<metadata xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dc="http://purl.org/dc/elements/1.1/" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:FamilyName>Emerson</dc:FamilyName><dc:GivenName>Everett H.</dc:GivenName></oai_dc:dc></metadata>';
157
'<metadata xmlns="http://www.openarchives.org/OAI/2.0/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><oai_dc:dc xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:dc="http://purl.org/dc/elements/1.1/" xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"><dc:FamilyName>Emerson</dc:FamilyName><dc:GivenName>Everett H.</dc:GivenName></oai_dc:dc></metadata>';
133
158
134
$r = HTTP::OAI::Record->new();
159
$r = HTTP::OAI::Record->new();
135
$r->metadata( HTTP::OAI::Metadata->new( dom => $auth ) );
160
$r->metadata( HTTP::OAI::Metadata->new( dom => $auth ) );
Lines 145-151 $status = $harvester->processRecord($r); Link Here
145
is( $status, 'updated', 'Authority with no date is updated' );
170
is( $status, 'updated', 'Authority with no date is updated' );
146
171
147
$status = $harvester->processRecord($r);
172
$status = $harvester->processRecord($r);
148
is( $status, 'updated', 'Authority with no date is updated even without force' );
173
is( $status, 'updated',
174
    'Authority with no date is updated even without force' );
149
175
150
$r->header->identifier('oai:myarchive.org:oid-162');
176
$r->header->identifier('oai:myarchive.org:oid-162');
151
$r->header->datestamp('2017-05-10T09:18:13Z');
177
$r->header->datestamp('2017-05-10T09:18:13Z');
Lines 156-170 is( $status, 'added', 'Authority is added' ); Link Here
156
$status = $harvester->processRecord($r);
182
$status = $harvester->processRecord($r);
157
is( $status, 'updated', 'When force is used, authority is updated' );
183
is( $status, 'updated', 'When force is used, authority is updated' );
158
184
159
$harvester = Koha::OAI::Client::Harvester->new( { server => $new_oai_auth, verbose => 1, days => 1, force => 0 } );
185
$harvester = Koha::OAI::Client::Harvester->new(
160
$status    = $harvester->processRecord($r);
186
    { server => $new_oai_auth, verbose => 1, days => 1, force => 0 } );
161
is( $status, 'skipped', 'When force is not used, authority is skipped (already up to date)' );
187
$status = $harvester->processRecord($r);
188
is( $status, 'skipped',
189
    'When force is not used, authority is skipped (already up to date)' );
162
190
163
$r->header->datestamp('2018-05-10T09:18:13Z');
191
$r->header->datestamp('2018-05-10T09:18:13Z');
164
$status = $harvester->processRecord($r);
192
$status = $harvester->processRecord($r);
165
is( $status, 'updated', 'When force is not used, authority is updated if newer' );
193
is( $status, 'updated',
194
    'When force is not used, authority is updated if newer' );
166
195
167
my $imported_authority = Koha::Import::OAI::Authorities->find( { identifier => 'oai:myarchive.org:oid-162' } );
196
my $imported_authority = Koha::Import::OAI::Authorities->find(
197
    { identifier => 'oai:myarchive.org:oid-162' } );
168
$imported_authority->update(
198
$imported_authority->update(
169
    {
199
    {
170
        datestamp => $added_datestamp,
200
        datestamp => $added_datestamp,
Lines 174-191 $imported_authority->update( Link Here
174
$r->header->datestamp(undef);
204
$r->header->datestamp(undef);
175
205
176
$status             = $harvester->processRecord($r);
206
$status             = $harvester->processRecord($r);
177
$imported_authority = Koha::Import::OAI::Authorities->find( { identifier => 'oai:myarchive.org:oid-162' } );
207
$imported_authority = Koha::Import::OAI::Authorities->find(
178
$updated_datestamp  = $imported_authority->datestamp;
208
    { identifier => 'oai:myarchive.org:oid-162' } );
179
isnt(
209
$updated_datestamp = $imported_authority->datestamp;
180
    $added_datestamp, $updated_datestamp,
210
isnt( $added_datestamp, $updated_datestamp,
181
    'local datestamp is updated even if there is no datestamp in incoming authority'
211
'local datestamp is updated even if there is no datestamp in incoming authority'
182
);
212
);
183
213
184
$r->header->status('deleted');
214
$r->header->status('deleted');
185
$status = $harvester->processRecord($r);
215
$status = $harvester->processRecord($r);
186
is( $status, 'deleted', 'When an authority is marked to be deleted, status is deleted' );
216
is( $status, 'deleted',
217
    'When an authority is marked to be deleted, status is deleted' );
187
218
188
$imported_record = Koha::Import::OAI::Biblios->find( { identifier => 'oai:myarchive.org:oid-162' } );
219
$imported_record = Koha::Import::OAI::Biblios->find(
220
    { identifier => 'oai:myarchive.org:oid-162' } );
189
is( $imported_record, undef, 'Authority has been deleted' );
221
is( $imported_record, undef, 'Authority has been deleted' );
190
222
191
$status = $harvester->processRecord($r);
223
$status = $harvester->processRecord($r);
192
- 

Return to bug 35659