|
Lines 24-38
use C4::Koha;
Link Here
|
| 24 |
use C4::Debug; |
24 |
use C4::Debug; |
| 25 |
use C4::Charset; |
25 |
use C4::Charset; |
| 26 |
use C4::Items; |
26 |
use C4::Items; |
|
|
27 |
use YAML; |
| 27 |
use Unicode::Normalize; |
28 |
use Unicode::Normalize; |
| 28 |
use Time::HiRes qw(gettimeofday); |
29 |
use Time::HiRes qw(gettimeofday); |
| 29 |
use Getopt::Long; |
30 |
use Getopt::Long; |
| 30 |
use IO::File; |
31 |
use IO::File; |
| 31 |
use Pod::Usage; |
32 |
use Pod::Usage; |
| 32 |
|
33 |
|
| 33 |
binmode STDOUT, ':encoding(UTF-8)'; |
34 |
use open qw( :std :utf8 ); |
|
|
35 |
binmode( STDOUT, ":utf8" ); |
| 34 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
36 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
| 35 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
37 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
|
|
38 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); |
| 36 |
my $cleanisbn = 1; |
39 |
my $cleanisbn = 1; |
| 37 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
40 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
| 38 |
my $framework = ''; |
41 |
my $framework = ''; |
|
Lines 46-52
GetOptions(
Link Here
|
| 46 |
'o|offset:f' => \$offset, |
49 |
'o|offset:f' => \$offset, |
| 47 |
'h' => \$version, |
50 |
'h' => \$version, |
| 48 |
'd' => \$delete, |
51 |
'd' => \$delete, |
| 49 |
't' => \$test_parameter, |
52 |
't|test' => \$test_parameter, |
| 50 |
's' => \$skip_marc8_conversion, |
53 |
's' => \$skip_marc8_conversion, |
| 51 |
'c:s' => \$char_encoding, |
54 |
'c:s' => \$char_encoding, |
| 52 |
'v:s' => \$verbose, |
55 |
'v:s' => \$verbose, |
|
Lines 56-77
GetOptions(
Link Here
|
| 56 |
'k|keepids:s' => \$keepids, |
59 |
'k|keepids:s' => \$keepids, |
| 57 |
'b|biblios' => \$biblios, |
60 |
'b|biblios' => \$biblios, |
| 58 |
'a|authorities' => \$authorities, |
61 |
'a|authorities' => \$authorities, |
|
|
62 |
'authtypes:s' => \$authtypes, |
| 63 |
'filter=s@' => \$filters, |
| 64 |
'insert' => \$insert, |
| 65 |
'update' => \$update, |
| 66 |
'all' => \$all, |
| 59 |
'match=s@' => \$match, |
67 |
'match=s@' => \$match, |
| 60 |
'i|isbn' => \$isbn_check, |
68 |
'i|isbn' => \$isbn_check, |
| 61 |
'x:s' => \$sourcetag, |
69 |
'x:s' => \$sourcetag, |
| 62 |
'y:s' => \$sourcesubfield, |
70 |
'y:s' => \$sourcesubfield, |
| 63 |
'idmap:s' => \$idmapfl, |
71 |
'idmap:s' => \$idmapfl, |
| 64 |
'cleanisbn!' => \$cleanisbn, |
72 |
'cleanisbn!' => \$cleanisbn, |
|
|
73 |
'yaml:s' => \$yamlfile, |
| 65 |
'dedupbarcode' => \$dedup_barcode, |
74 |
'dedupbarcode' => \$dedup_barcode, |
| 66 |
'framework=s' => \$framework, |
75 |
'framework=s' => \$framework, |
| 67 |
); |
76 |
); |
| 68 |
$biblios=!$authorities||$biblios; |
77 |
$biblios ||= !$authorities; |
|
|
78 |
$insert ||= !$update; |
| 79 |
|
| 80 |
if ($all) { |
| 81 |
$insert = 1; |
| 82 |
$update = 1; |
| 83 |
} |
| 69 |
|
84 |
|
| 70 |
if ($version || ($input_marc_file eq '')) { |
85 |
if ($version || ($input_marc_file eq '')) { |
| 71 |
pod2usage( -verbose => 2 ); |
86 |
pod2usage( -verbose => 2 ); |
| 72 |
exit; |
87 |
exit; |
| 73 |
} |
88 |
} |
| 74 |
|
89 |
|
|
|
90 |
my $dbh = C4::Context->dbh; |
| 91 |
my $heading_fields=get_heading_fields(); |
| 92 |
|
| 75 |
if (defined $idmapfl) { |
93 |
if (defined $idmapfl) { |
| 76 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
94 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
| 77 |
} |
95 |
} |
|
Lines 81-88
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
Link Here
|
| 81 |
$sourcesubfield="a"; |
99 |
$sourcesubfield="a"; |
| 82 |
} |
100 |
} |
| 83 |
|
101 |
|
| 84 |
my $dbh = C4::Context->dbh; |
|
|
| 85 |
|
| 86 |
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & |
102 |
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & |
| 87 |
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user. |
103 |
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user. |
| 88 |
my $CataloguingLog = C4::Context->preference('CataloguingLog'); |
104 |
my $CataloguingLog = C4::Context->preference('CataloguingLog'); |
|
Lines 142-148
$batch->warnings_off();
Link Here
|
| 142 |
$batch->strict_off(); |
158 |
$batch->strict_off(); |
| 143 |
my $i=0; |
159 |
my $i=0; |
| 144 |
my $commitnum = $commit ? $commit : 50; |
160 |
my $commitnum = $commit ? $commit : 50; |
| 145 |
|
161 |
my $yamlhash; |
| 146 |
|
162 |
|
| 147 |
# Skip file offset |
163 |
# Skip file offset |
| 148 |
if ( $offset ) { |
164 |
if ( $offset ) { |
|
Lines 199-204
RECORD: while ( ) {
Link Here
|
| 199 |
next RECORD; |
215 |
next RECORD; |
| 200 |
} |
216 |
} |
| 201 |
} |
217 |
} |
|
|
218 |
SetUTF8Flag($record); |
| 202 |
my $isbn; |
219 |
my $isbn; |
| 203 |
# remove trailing - in isbn (only for biblios, of course) |
220 |
# remove trailing - in isbn (only for biblios, of course) |
| 204 |
if ($biblios && $cleanisbn) { |
221 |
if ($biblios && $cleanisbn) { |
|
Lines 212-256
RECORD: while ( ) {
Link Here
|
| 212 |
} |
229 |
} |
| 213 |
my $id; |
230 |
my $id; |
| 214 |
# search for duplicates (based on Local-number) |
231 |
# search for duplicates (based on Local-number) |
| 215 |
if ($match){ |
232 |
my $originalid; |
| 216 |
require C4::Search; |
233 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
| 217 |
my $query=build_query($match,$record); |
234 |
if ($match) { |
| 218 |
my $server=($authorities?'authorityserver':'biblioserver'); |
235 |
require C4::Search; |
| 219 |
my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
236 |
my $query = build_query( $match, $record ); |
| 220 |
die "unable to search the database for duplicates : $error" if (defined $error); |
237 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
| 221 |
#warn "$query $server : $totalhits"; |
238 |
$debug && warn $query; |
| 222 |
if ( @{$results} == 1 ){ |
239 |
my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
| 223 |
my $marcrecord = MARC::File::USMARC::decode($results->[0]); |
240 |
die "unable to search the database for duplicates : $error" if ( defined $error ); |
| 224 |
$id=GetRecordId($marcrecord,$tagid,$subfieldid); |
241 |
$debug && warn "$query $server : $totalhits"; |
| 225 |
} |
242 |
if ( $results && scalar(@$results) == 1 ) { |
| 226 |
elsif ( @{$results} > 1){ |
243 |
my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); |
| 227 |
$debug && warn "more than one match for $query"; |
244 |
SetUTF8Flag($marcrecord); |
| 228 |
} |
245 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
| 229 |
else { |
246 |
if ( $authorities && $marcFlavour ) { |
| 230 |
$debug && warn "nomatch for $query"; |
247 |
#Skip if authority in database is the same as the on in database |
| 231 |
} |
248 |
if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { |
|
|
249 |
if ($yamlfile) { |
| 250 |
$yamlhash->{$originalid}->{'authid'} = $id; |
| 251 |
|
| 252 |
# we recover all subfields of the heading authorities |
| 253 |
my @subfields; |
| 254 |
foreach my $field ( $marcrecord->field("2..") ) { |
| 255 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
| 256 |
} |
| 257 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
| 258 |
} |
| 259 |
next; |
| 260 |
} |
| 261 |
} |
| 262 |
} elsif ( $results && scalar(@$results) > 1 ) { |
| 263 |
$debug && warn "more than one match for $query"; |
| 264 |
} else { |
| 265 |
$debug && warn "nomatch for $query"; |
| 266 |
} |
| 267 |
} |
| 268 |
if ($keepids && $originalid) { |
| 269 |
my $storeidfield; |
| 270 |
if ( length($keepids) == 3 ) { |
| 271 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
| 272 |
} else { |
| 273 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
| 274 |
} |
| 275 |
$record->insert_fields_ordered($storeidfield); |
| 276 |
$record->delete_field( $record->field($tagid) ); |
| 232 |
} |
277 |
} |
| 233 |
my $originalid; |
278 |
foreach my $stringfilter (@$filters) { |
| 234 |
if ($keepids){ |
279 |
if ( length($stringfilter) == 3 ) { |
| 235 |
$originalid=GetRecordId($record,$tagid,$subfieldid); |
280 |
foreach my $field ( $record->field($stringfilter) ) { |
| 236 |
if ($originalid){ |
281 |
$record->delete_field($field); |
| 237 |
my $storeidfield; |
282 |
$debug && warn "removed : ", $field->as_string; |
| 238 |
if (length($keepids)==3){ |
283 |
} |
| 239 |
$storeidfield=MARC::Field->new($keepids,$originalid); |
284 |
} else { |
| 240 |
} |
285 |
my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) |
| 241 |
else { |
286 |
if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; |
| 242 |
$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); |
287 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
| 243 |
} |
288 |
foreach my $field ( $record->field($removetag) ) { |
| 244 |
$record->insert_fields_ordered($storeidfield); |
289 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
| 245 |
$record->delete_field($record->field($tagid)); |
290 |
$debug && warn "Potentially removed : ", $field->subfield($removesubfield); |
| 246 |
} |
291 |
} |
|
|
292 |
} |
| 293 |
} |
| 247 |
} |
294 |
} |
| 248 |
unless ($test_parameter) { |
295 |
unless ($test_parameter) { |
| 249 |
if ($authorities){ |
296 |
if ($authorities){ |
| 250 |
use C4::AuthoritiesMarc; |
297 |
use C4::AuthoritiesMarc; |
| 251 |
my $authtypecode=GuessAuthTypeCode($record); |
298 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
| 252 |
my $authid= ($id?$id:GuessAuthId($record)); |
299 |
my $authid= ($id?$id:GuessAuthId($record)); |
| 253 |
if ($authid && GetAuthority($authid)){ |
300 |
if ($authid && GetAuthority($authid) && $update ){ |
| 254 |
## Authority has an id and is in database : Replace |
301 |
## Authority has an id and is in database : Replace |
| 255 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
302 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
| 256 |
if ($@){ |
303 |
if ($@){ |
|
Lines 283-288
RECORD: while ( ) {
Link Here
|
| 283 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
330 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
| 284 |
} |
331 |
} |
| 285 |
} |
332 |
} |
|
|
333 |
if ($yamlfile) { |
| 334 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
| 335 |
my @subfields; |
| 336 |
foreach my $field ( $record->field("2..") ) { |
| 337 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
| 338 |
} |
| 339 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
| 340 |
} |
| 286 |
} |
341 |
} |
| 287 |
else { |
342 |
else { |
| 288 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
343 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
|
Lines 306-325
RECORD: while ( ) {
Link Here
|
| 306 |
} |
361 |
} |
| 307 |
# create biblio, unless we already have it ( either match or isbn ) |
362 |
# create biblio, unless we already have it ( either match or isbn ) |
| 308 |
if ($biblionumber) { |
363 |
if ($biblionumber) { |
| 309 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} |
364 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; |
| 310 |
} |
365 |
if ($update) { |
| 311 |
else |
366 |
eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; |
| 312 |
{ |
367 |
if ($@) { |
| 313 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; |
368 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
|
|
369 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
| 370 |
next RECORD; |
| 371 |
} else { |
| 372 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
| 373 |
} |
| 374 |
} else { |
| 375 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
| 376 |
} |
| 377 |
} else { |
| 378 |
if ($insert) { |
| 379 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; |
| 380 |
if ($@) { |
| 381 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
| 382 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
| 383 |
next RECORD; |
| 384 |
} else { |
| 385 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
| 386 |
} |
| 387 |
} else { |
| 388 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
| 389 |
} |
| 314 |
} |
390 |
} |
| 315 |
if ( $@ ) { |
|
|
| 316 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
| 317 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); |
| 318 |
next RECORD; |
| 319 |
} |
| 320 |
else{ |
| 321 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); |
| 322 |
} |
| 323 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
391 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
| 324 |
my $error_adding = $@; |
392 |
my $error_adding = $@; |
| 325 |
# Work on a clone so that if there are real errors, we can maybe |
393 |
# Work on a clone so that if there are real errors, we can maybe |
|
Lines 384-389
RECORD: while ( ) {
Link Here
|
| 384 |
if ($#{ $errors_ref } > -1) { |
452 |
if ($#{ $errors_ref } > -1) { |
| 385 |
report_item_errors($biblionumber, $errors_ref); |
453 |
report_item_errors($biblionumber, $errors_ref); |
| 386 |
} |
454 |
} |
|
|
455 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
| 387 |
} |
456 |
} |
| 388 |
$dbh->commit() if (0 == $i % $commitnum); |
457 |
$dbh->commit() if (0 == $i % $commitnum); |
| 389 |
} |
458 |
} |
|
Lines 407-412
if ($logfile){
Link Here
|
| 407 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
476 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
| 408 |
$loghandle->close; |
477 |
$loghandle->close; |
| 409 |
} |
478 |
} |
|
|
479 |
if ($yamlfile) { |
| 480 |
open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; |
| 481 |
print YAML Dump($yamlhash); |
| 482 |
} |
| 410 |
exit 0; |
483 |
exit 0; |
| 411 |
|
484 |
|
| 412 |
sub GetRecordId{ |
485 |
sub GetRecordId{ |
|
Lines 462-470
sub report_item_errors {
Link Here
|
| 462 |
} |
535 |
} |
| 463 |
sub printlog{ |
536 |
sub printlog{ |
| 464 |
my $logelements=shift; |
537 |
my $logelements=shift; |
| 465 |
print $loghandle join (";",@$logelements{qw<id op status>}),"\n"; |
538 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
|
|
539 |
} |
| 540 |
sub get_heading_fields{ |
| 541 |
my $headingfields; |
| 542 |
if ($authtypes){ |
| 543 |
$headingfields=YAML::LoadFile($authtypes); |
| 544 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
| 545 |
$debug && warn YAML::Dump($headingfields); |
| 546 |
} |
| 547 |
unless ($headingfields){ |
| 548 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
| 549 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
| 550 |
} |
| 551 |
return $headingfields; |
| 466 |
} |
552 |
} |
| 467 |
|
|
|
| 468 |
|
553 |
|
| 469 |
=head1 NAME |
554 |
=head1 NAME |
| 470 |
|
555 |
|
|
Lines 526-532
The I<NUMBER> of records to wait before performing a 'commit' operation
Link Here
|
| 526 |
|
611 |
|
| 527 |
File logs actions done for each record and their status into file |
612 |
File logs actions done for each record and their status into file |
| 528 |
|
613 |
|
| 529 |
=item B<-t> |
614 |
=item B<-t, -test> |
| 530 |
|
615 |
|
| 531 |
Test mode: parses the file, saying what he would do, but doing nothing. |
616 |
Test mode: parses the file, saying what he would do, but doing nothing. |
| 532 |
|
617 |
|
|
Lines 549-554
biblioitems, items
Link Here
|
| 549 |
|
634 |
|
| 550 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
635 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
| 551 |
|
636 |
|
|
|
637 |
=item B<-authtypes> |
| 638 |
|
| 639 |
file yamlfile with authoritiesTypes and distinguishable record field in order |
| 640 |
to store the correct authtype |
| 641 |
|
| 642 |
=item B<-yaml> |
| 643 |
|
| 644 |
yaml file format a yaml file with ids |
| 645 |
keepids field store ids in field (usefull for authorities, |
| 646 |
where 001 contains the authid for Koha, that can contain a very valuable |
| 647 |
info for authorities coming from LOC or BNF. useless for biblios probably |
| 648 |
|
| 649 |
=item B<-insert> |
| 650 |
|
| 651 |
if set, only insert when possible |
| 652 |
|
| 653 |
=item B<-update> |
| 654 |
|
| 655 |
if set, only updates (any biblio should have a matching record) |
| 656 |
|
| 657 |
=item B<-all> |
| 658 |
|
| 659 |
if set, do whatever is required |
| 660 |
|
| 552 |
=item B<-k, -keepids>=<FIELD> |
661 |
=item B<-k, -keepids>=<FIELD> |
| 553 |
|
662 |
|
| 554 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
663 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
| 555 |
- |
|
|