|
Line 0
Link Here
|
| 0 |
- |
1 |
#!/usr/bin/perl |
|
|
2 |
|
| 3 |
# This file is part of Koha. |
| 4 |
# |
| 5 |
# Copyright (C) 2025 Dataly Tech |
| 6 |
# |
| 7 |
# Koha is free software; you can redistribute it and/or modify it |
| 8 |
# under the terms of the GNU General Public License as published by |
| 9 |
# the Free Software Foundation; either version 3 of the License, or |
| 10 |
# (at your option) any later version. |
| 11 |
# |
| 12 |
# Koha is distributed in the hope that it will be useful, but |
| 13 |
# WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 |
# GNU General Public License for more details. |
| 16 |
# |
| 17 |
# You should have received a copy of the GNU General Public License |
| 18 |
# along with Koha; if not, see <http://www.gnu.org/licenses>. |
| 19 |
|
| 20 |
use Modern::Perl; |
| 21 |
|
| 22 |
use C4::AuthoritiesMarc qw( GetAuthority ModAuthority ); |
| 23 |
use C4::Context qw( preference ); |
| 24 |
use Koha::Authorities; |
| 25 |
use Koha::Caches; |
| 26 |
use Koha::ExternalContent::OCLC; |
| 27 |
use Koha::Script; |
| 28 |
|
| 29 |
use Getopt::Long qw( GetOptions :config no_auto_abbrev no_ignore_case ); |
| 30 |
use JSON qw( from_json ); |
| 31 |
use LWP::UserAgent; |
| 32 |
use MARC::Record; |
| 33 |
use Pod::Usage qw( pod2usage ); |
| 34 |
|
| 35 |
binmode( STDOUT, ':encoding(UTF-8)' ); |
| 36 |
|
| 37 |
my @authtypecodes; |
| 38 |
my $confirm = 0; |
| 39 |
my $help; |
| 40 |
my $force = 0; |
| 41 |
my $language; |
| 42 |
my $man; |
| 43 |
my $verbose = 0; |
| 44 |
my $where; |
| 45 |
|
| 46 |
GetOptions( |
| 47 |
'a|authtypecode=s' => \@authtypecodes, |
| 48 |
'c|confirm' => \$confirm, |
| 49 |
'h|help' => \$help, |
| 50 |
'f|force' => \$force, |
| 51 |
'l|language=s{1}' => \$language, |
| 52 |
'man' => \$man, |
| 53 |
'v|verbose' => \$verbose, |
| 54 |
'w|where=s' => \$where, |
| 55 |
) or pod2usage(1); |
| 56 |
|
| 57 |
pod2usage(1) if $help; |
| 58 |
pod2usage( -verbose => 2 ) if $man; |
| 59 |
|
| 60 |
my %languages = ( |
| 61 |
'de' => 'German', |
| 62 |
'en' => 'English', |
| 63 |
'fr' => 'French', |
| 64 |
'it' => 'Italian', |
| 65 |
'no' => 'Norwegian', |
| 66 |
'sv' => 'Swedish', |
| 67 |
); |
| 68 |
|
| 69 |
my $langcodes = join '|', sort keys %languages; |
| 70 |
|
| 71 |
if ( $language and $language !~ /^($langcodes)$/ ) { |
| 72 |
print "Incorrect language code passed, must be one of: $langcodes\n"; |
| 73 |
exit; |
| 74 |
} |
| 75 |
|
| 76 |
print "Running in dry-run mode as -c/--confirm is not passed\n" unless $confirm; |
| 77 |
|
| 78 |
my $cache = Koha::Caches->get_instance; |
| 79 |
my $client_id = C4::Context->preference('OCLCAPIWSKeyClientID'); |
| 80 |
my $client_secret = C4::Context->preference('OCLCAPIWSKeyClientSecret'); |
| 81 |
my $pref_language = C4::Context->preference('OCLCDeweyLinkedDataLanguage'); |
| 82 |
my $marcflavour = C4::Context->preference('marcflavour'); |
| 83 |
my $scope = 'deweyLinkedData'; |
| 84 |
|
| 85 |
$verbose and print "OCLCDeweyLinkedDataLanguage set to '$pref_language' ($languages{$pref_language})\n"; |
| 86 |
$verbose and print "Language overriden and set to '$language' ($languages{$language})\n" if $language; |
| 87 |
$verbose and print "Option --force passed, will overwrite subfield data\n" if $force; |
| 88 |
|
| 89 |
my ( $access_token, $msg ); |
| 90 |
if ( $client_id and $client_secret ) { |
| 91 |
$verbose and print "Found OCLC Dewey Linked Data API Client ID and Secret\n"; |
| 92 |
$verbose and print "Requesting an access token from OCLC's OAuth server\n"; |
| 93 |
( $access_token, $msg ) = Koha::ExternalContent::OCLC->get_access_token( |
| 94 |
{ |
| 95 |
scope => $scope, |
| 96 |
} |
| 97 |
); |
| 98 |
if ( defined $access_token ) { |
| 99 |
$verbose and print "Got token '$access_token' via $msg\n"; |
| 100 |
} else { |
| 101 |
$verbose and print "Cannot obtain API token ($msg)\n"; |
| 102 |
exit; |
| 103 |
} |
| 104 |
} else { |
| 105 |
print "Cannot request an API access token from OCLC's OAuth server\n" |
| 106 |
. "An OCLC Dewey Linked Data API Client ID and Secret are required\n" |
| 107 |
. "Please fill in the OCLCAPIWSKeyClientID and OCLCAPIWSKeyClientSecret\n" |
| 108 |
. "System Preferences, then try again.\n"; |
| 109 |
exit; |
| 110 |
} |
| 111 |
|
| 112 |
$verbose and print "Fetching auth types\n"; |
| 113 |
my $params = undef; |
| 114 |
if (@authtypecodes) { |
| 115 |
$params = { authtypecode => { -in => \@authtypecodes } }; |
| 116 |
} |
| 117 |
my @auth_types = Koha::Authority::Types->search($params)->as_list; |
| 118 |
$verbose and print "Done fetching auth types\n"; |
| 119 |
|
| 120 |
my ( $ddc_fields_regex, $ddc_url_subfield ); |
| 121 |
if ( $marcflavour eq 'MARC21' ) { |
| 122 |
$ddc_fields_regex = '082|083'; |
| 123 |
$ddc_url_subfield = '1'; |
| 124 |
} elsif ( $marcflavour eq 'UNIMARC' ) { |
| 125 |
$ddc_fields_regex = '676'; |
| 126 |
$ddc_url_subfield = 'R'; |
| 127 |
} |
| 128 |
my $ddc_descr_subfield = 'c'; |
| 129 |
|
| 130 |
# The regex below Covers DDC numbers like 270, 248.4819, T4--313 |
| 131 |
my $ddc_regex = '^((T([1-6])--([0-9]+)))$|^([0-9]{3})$|^(([0-9]{3})\.([0-9]+))$'; |
| 132 |
|
| 133 |
foreach my $authtype (@auth_types) { |
| 134 |
|
| 135 |
my $authtypecode = $authtype->authtypecode; |
| 136 |
$verbose and print "Fetching authorities of type '$authtypecode'\n"; |
| 137 |
my $authorities = Koha::Authorities->search( |
| 138 |
{ authtypecode => $authtypecode }, |
| 139 |
{ order_by => { -asc => 'authid' } } |
| 140 |
); |
| 141 |
$authorities = $authorities->search( \$where ) if $where; |
| 142 |
|
| 143 |
my $count = $authorities->count; |
| 144 |
$verbose and print "Found $count authorit" . ( $count == 1 ? 'y' : 'ies' ) . "\n"; |
| 145 |
|
| 146 |
while ( my $authority = $authorities->next ) { |
| 147 |
process_authority($authority); |
| 148 |
} |
| 149 |
|
| 150 |
} |
| 151 |
|
| 152 |
sub process_authority { |
| 153 |
|
| 154 |
my ($authority) = @_; |
| 155 |
|
| 156 |
my $authid = $authority->authid; |
| 157 |
my $record = GetAuthority($authid); |
| 158 |
my @fields = $record->field($ddc_fields_regex); |
| 159 |
my $modified = 0; |
| 160 |
|
| 161 |
foreach my $field (@fields) { |
| 162 |
|
| 163 |
my ( $ddc_range_begin, $ddc_range_end ); |
| 164 |
|
| 165 |
$ddc_range_begin = $field && $field->subfield('a'); |
| 166 |
if ( $ddc_range_begin and $ddc_range_begin !~ /$ddc_regex/ ) { |
| 167 |
$verbose and print "auth $authid: invalid DDC number '$ddc_range_begin' in subfield \$a\n"; |
| 168 |
next; |
| 169 |
} |
| 170 |
|
| 171 |
if ( ( $marcflavour eq 'MARC21' and $field->tag eq '083' ) |
| 172 |
or ( $marcflavour eq 'UNIMARC' and $field->tag eq '676' ) ) |
| 173 |
{ |
| 174 |
$ddc_range_end = $field && $field->subfield('b'); |
| 175 |
if ( $ddc_range_end and $ddc_range_end !~ /$ddc_regex/ ) { |
| 176 |
$verbose and print "auth $authid: invalid DDC number '$ddc_range_end' in subfield \$b\n"; |
| 177 |
next; |
| 178 |
} |
| 179 |
} |
| 180 |
|
| 181 |
my $rec_ddc = |
| 182 |
$ddc_range_end |
| 183 |
? $ddc_range_begin . '-' . $ddc_range_end |
| 184 |
: $ddc_range_begin |
| 185 |
if $ddc_range_begin; |
| 186 |
|
| 187 |
my $rec_ddc_url = $field && $field->subfield($ddc_url_subfield); |
| 188 |
my $rec_ddc_descr = $field && $field->subfield($ddc_descr_subfield); |
| 189 |
|
| 190 |
if ( $rec_ddc and $rec_ddc_url and $rec_ddc_descr ) { |
| 191 |
$verbose and print "auth $authid: found DDC '$rec_ddc', LD URL '$rec_ddc_url'\n"; |
| 192 |
$verbose and print "auth $authid: found DDC '$rec_ddc', LD Description '$rec_ddc_descr'\n"; |
| 193 |
$verbose and print "auth $authid: skipping this field, as all data is present\n" unless $force; |
| 194 |
$verbose and print "auth $authid: you can use --force to override this behavior\n" unless $force; |
| 195 |
next unless $force; |
| 196 |
} |
| 197 |
|
| 198 |
if ($rec_ddc) { |
| 199 |
|
| 200 |
$verbose and print "auth $authid: found DDC '$rec_ddc', searching for URI/URL/Description\n"; |
| 201 |
|
| 202 |
# Trim leading 'T' (uppercase 't') from DDC number |
| 203 |
$rec_ddc =~ s/^T//; |
| 204 |
|
| 205 |
my ( $ddc_uri, $ddc_url, $ddc_descr ); |
| 206 |
|
| 207 |
################################################################## |
| 208 |
# Get DDC URI (retry automatically if the token expired) |
| 209 |
################################################################## |
| 210 |
( $ddc_uri, $msg ) = _get_ddc_uri( $access_token, $rec_ddc ); |
| 211 |
if ( defined $ddc_uri ) { |
| 212 |
$verbose and print "auth $authid: found LD URI '$ddc_uri' via $msg\n"; |
| 213 |
} elsif ( $msg eq 'Unauthorized' ) { |
| 214 |
$verbose and print "auth $authid: cannot obtain LD URI ($msg)\n"; |
| 215 |
$verbose and print "auth $authid: our API access token has expired\n"; |
| 216 |
$verbose and print "auth $authid: removing the expired token from cache\n"; |
| 217 |
$cache->clear_from_cache("OCLC_access_token-$scope"); |
| 218 |
$verbose and print "auth $authid: requesting a new access token\n"; |
| 219 |
( $access_token, $msg ) = Koha::ExternalContent::OCLC->get_access_token( |
| 220 |
{ |
| 221 |
scope => $scope, |
| 222 |
} |
| 223 |
); |
| 224 |
if ( defined $access_token ) { |
| 225 |
$verbose and print "got renewed access token '$access_token' via $msg\n"; |
| 226 |
$verbose and print "retrying LD URI fetch for DDC '$rec_ddc'\n"; |
| 227 |
( $ddc_uri, $msg ) = _get_ddc_uri( $access_token, $rec_ddc ); |
| 228 |
} else { |
| 229 |
$verbose and print "Cannot obtain API token ($msg)\n"; |
| 230 |
exit; |
| 231 |
} |
| 232 |
redo; |
| 233 |
} elsif ( $msg eq 'not found' ) { |
| 234 |
$verbose and print "auth $authid: no LD URI found for DDC '$rec_ddc'\n"; |
| 235 |
|
| 236 |
# skip this field, as no URI was returned (no further API lookups needed) |
| 237 |
$verbose and print "auth $authid: skipping this field\n"; |
| 238 |
next; |
| 239 |
} |
| 240 |
|
| 241 |
################################################################## |
| 242 |
# Get DDC URL (retry automatically if the token expired) |
| 243 |
################################################################## |
| 244 |
( $ddc_url, $msg ) = _get_ddc_url( $access_token, $rec_ddc ); |
| 245 |
if ( defined $ddc_url ) { |
| 246 |
$verbose and print "auth $authid: found LD URL '$ddc_url' via $msg\n"; |
| 247 |
} elsif ( $msg eq 'Unauthorized' ) { |
| 248 |
$verbose and print "auth $authid: cannot obtain LD URL ($msg)\n"; |
| 249 |
$verbose and print "auth $authid: our API access token has expired\n"; |
| 250 |
$verbose and print "auth $authid: removing the expired token from cache\n"; |
| 251 |
$cache->clear_from_cache("OCLC_access_token-$scope"); |
| 252 |
$verbose and print "auth $authid: requesting a new access token\n"; |
| 253 |
( $access_token, $msg ) = Koha::ExternalContent::OCLC->get_access_token( |
| 254 |
{ |
| 255 |
scope => $scope, |
| 256 |
} |
| 257 |
); |
| 258 |
if ( defined $access_token ) { |
| 259 |
$verbose and print "Got renewed access token '$access_token' via $msg\n"; |
| 260 |
$verbose and print "retrying LD URL fetch for DDC '$rec_ddc'\n"; |
| 261 |
( $ddc_url, $msg ) = _get_ddc_url( $access_token, $rec_ddc ); |
| 262 |
} else { |
| 263 |
$verbose and print "Cannot obtain API token ($msg)\n"; |
| 264 |
exit; |
| 265 |
} |
| 266 |
redo; |
| 267 |
} elsif ( $msg eq 'not found' ) { |
| 268 |
$verbose and print "auth $authid: no LD URL found for DDC '$rec_ddc'\n"; |
| 269 |
} |
| 270 |
|
| 271 |
################################################################## |
| 272 |
# Get DDC Description (retry automatically if the token expired) |
| 273 |
################################################################## |
| 274 |
my $lang = $language ? $language : undef; |
| 275 |
( $ddc_descr, $msg ) = _get_ddc_descr( $access_token, $ddc_uri, $lang ); |
| 276 |
if ( defined $ddc_descr ) { |
| 277 |
$verbose and print "auth $authid: found LD Description '$ddc_descr' via $msg\n"; |
| 278 |
} elsif ( $msg eq 'Unauthorized' ) { |
| 279 |
$verbose and print "auth $authid: cannot obtain LD Description ($msg)\n"; |
| 280 |
$verbose and print "auth $authid: our API access token has expired\n"; |
| 281 |
$verbose and print "auth $authid: removing the expired token from cache\n"; |
| 282 |
$cache->clear_from_cache("OCLC_access_token-$scope"); |
| 283 |
$verbose and print "auth $authid: requesting a new access token\n"; |
| 284 |
( $access_token, $msg ) = Koha::ExternalContent::OCLC->get_access_token( |
| 285 |
{ |
| 286 |
scope => $scope, |
| 287 |
} |
| 288 |
); |
| 289 |
if ( defined $access_token ) { |
| 290 |
$verbose and print "Got renewed access token '$access_token' via $msg\n"; |
| 291 |
$verbose and print "retrying LD Description fetch for DDC '$rec_ddc'\n"; |
| 292 |
( $ddc_descr, $msg ) = _get_ddc_descr( $access_token, $ddc_uri, $lang ); |
| 293 |
} else { |
| 294 |
$verbose and print "Cannot obtain API token ($msg)\n"; |
| 295 |
exit; |
| 296 |
} |
| 297 |
redo; |
| 298 |
} elsif ( $msg eq 'not found' ) { |
| 299 |
$verbose and print "auth $authid: no LD Description found for DDC '$rec_ddc'\n"; |
| 300 |
} |
| 301 |
|
| 302 |
if ( $confirm or $force ) { |
| 303 |
$field->update( $ddc_url_subfield => $ddc_url ); |
| 304 |
|
| 305 |
# MARC21 authority field 082 does not have a $c subfield, |
| 306 |
# so we only update the description if the current field |
| 307 |
# is 083 (MARC21) or 676 (UNIMARC) |
| 308 |
if ( $field->tag eq '083' or $field->tag eq '676' ) { |
| 309 |
$field->update( $ddc_descr_subfield => $ddc_descr ); |
| 310 |
} |
| 311 |
$modified = 1; |
| 312 |
} |
| 313 |
|
| 314 |
} |
| 315 |
|
| 316 |
} |
| 317 |
|
| 318 |
if ($modified) { |
| 319 |
my $auth_count = $authority->get_usage_count; |
| 320 |
$verbose and print "auth $authid: used in $auth_count bibliographic records\n"; |
| 321 |
$verbose and print "auth $authid: modified, calling ModAuthority()\n"; |
| 322 |
my $ret = ModAuthority( $authid, $record, $authority->authtypecode ); |
| 323 |
if ( $ret == $authid ) { |
| 324 |
$verbose and print "auth $authid: updated successfully\n"; |
| 325 |
} else { |
| 326 |
$verbose and print "auth $authid: update failure\n"; |
| 327 |
} |
| 328 |
} |
| 329 |
|
| 330 |
} |
| 331 |
|
| 332 |
sub _get_ddc_uri { |
| 333 |
my $access_token = shift; |
| 334 |
my $ddc_number = shift; |
| 335 |
my ( $ddc_uri, $msg ) = Koha::ExternalContent::OCLC->get_ddc_uri( |
| 336 |
{ |
| 337 |
access_token => $access_token, |
| 338 |
ddc_number => $ddc_number, |
| 339 |
} |
| 340 |
); |
| 341 |
return ( $ddc_uri, $msg ); |
| 342 |
} |
| 343 |
|
| 344 |
sub _get_ddc_url { |
| 345 |
my $access_token = shift; |
| 346 |
my $ddc_number = shift; |
| 347 |
my ( $ddc_url, $msg ) = Koha::ExternalContent::OCLC->get_ddc_url( |
| 348 |
{ |
| 349 |
access_token => $access_token, |
| 350 |
ddc_number => $ddc_number, |
| 351 |
} |
| 352 |
); |
| 353 |
return ( $ddc_url, $msg ); |
| 354 |
} |
| 355 |
|
| 356 |
sub _get_ddc_descr { |
| 357 |
my $access_token = shift; |
| 358 |
my $ddc_uri = shift; |
| 359 |
my $language = shift; |
| 360 |
my ( $ddc_descr, $msg ) = Koha::ExternalContent::OCLC->get_ddc_description( |
| 361 |
{ |
| 362 |
access_token => $access_token, |
| 363 |
ddc_uri => $ddc_uri, |
| 364 |
language => $language, |
| 365 |
} |
| 366 |
); |
| 367 |
return ( $ddc_descr, $msg ); |
| 368 |
} |
| 369 |
|
| 370 |
=head1 NAME |
| 371 |
|
| 372 |
fetch_oclc_dewey_linked_data.pl - Updates authority records with OCLC Dewey Linked Data |
| 373 |
|
| 374 |
=head1 SYNOPSIS |
| 375 |
|
| 376 |
B<fetch_oclc_dewey_linked_data.pl> |
| 377 |
[B<-a|--authtypecode ...>] |
| 378 |
[B<-c|--confirm>] |
| 379 |
[B<-h|--help>] |
| 380 |
[B<-f|--force>] |
| 381 |
[B<-l|--language de|en|fr|it|no|sv>] |
| 382 |
[B<--man>] |
| 383 |
[B<-v|--verbose>] |
| 384 |
[B<-w|--where ...>] |
| 385 |
|
| 386 |
=head1 DESCRIPTION |
| 387 |
|
| 388 |
This script automatically fetches Linked Data URLs corresponding to the |
| 389 |
DDC numbers stored in authority records, and stores them in the |
| 390 |
appropriate MARC subfield. It works for both MARC21 and UNIMARC. |
| 391 |
|
| 392 |
In order to use the OCLC Dewey Linked Data API an access token must be |
| 393 |
generated (this is handled internally by the script), which requires the |
| 394 |
OCLCAPIWSKeyClientID and OCLCAPIWSKeyClientSecret System Preferences to |
| 395 |
be filled in. |
| 396 |
|
| 397 |
=head1 OPTIONS |
| 398 |
|
| 399 |
=over |
| 400 |
|
| 401 |
=item B<-a|--authtypecode> |
| 402 |
|
| 403 |
Repeatable; selects the authtypecode(s) to use when searching for authorities. |
| 404 |
|
| 405 |
=item B<-c|--confirm> |
| 406 |
|
| 407 |
Makes the script actually modify any matching authorities, otherwise only a |
| 408 |
dry run will be performed. |
| 409 |
|
| 410 |
=item B<-f|--force> |
| 411 |
|
| 412 |
Enforces an overwrite of previously set MARC subfield values (specifically |
| 413 |
the Linked Data URL and Description). Use this option in combination with |
| 414 |
--language to re-fetch the DDC Descriptions from OCLC's API for a language |
| 415 |
different than OCLCDeweyLinkedDataLanguage. Use this option on its own to |
| 416 |
re-fetch DDC Descriptions for the language set in OCLCDeweyLinkedDataLanguage. |
| 417 |
Implies --confirm. |
| 418 |
|
| 419 |
=item B<-h|--help> |
| 420 |
|
| 421 |
Prints a brief help message. |
| 422 |
|
| 423 |
=item B<-l|--language> |
| 424 |
|
| 425 |
Overrides the value set by the OCLCDeweyLinkedDataLanguage System Preference. |
| 426 |
The argument to this option must be one of: de|en|fr|it|no|sv. |
| 427 |
|
| 428 |
=item B<--man> |
| 429 |
|
| 430 |
Prints a detailed help message. |
| 431 |
|
| 432 |
=item B<-v|--verbose> |
| 433 |
|
| 434 |
Highly recommended; prints verbose progress information. |
| 435 |
|
| 436 |
=item B<-w|--where> |
| 437 |
|
| 438 |
Passes an additional SQL 'WHERE' clause to the authority search, in order to |
| 439 |
limit the results returned. |
| 440 |
|
| 441 |
=back |
| 442 |
|
| 443 |
=head1 AUTHOR |
| 444 |
|
| 445 |
Andreas Roussos <a.roussos@dataly.gr> |
| 446 |
|
| 447 |
=cut |