Lines 24-38
use C4::Koha;
Link Here
|
24 |
use C4::Debug; |
24 |
use C4::Debug; |
25 |
use C4::Charset; |
25 |
use C4::Charset; |
26 |
use C4::Items; |
26 |
use C4::Items; |
|
|
27 |
use YAML; |
27 |
use Unicode::Normalize; |
28 |
use Unicode::Normalize; |
28 |
use Time::HiRes qw(gettimeofday); |
29 |
use Time::HiRes qw(gettimeofday); |
29 |
use Getopt::Long; |
30 |
use Getopt::Long; |
30 |
use IO::File; |
31 |
use IO::File; |
31 |
use Pod::Usage; |
32 |
use Pod::Usage; |
32 |
|
33 |
|
33 |
binmode STDOUT, ':encoding(UTF-8)'; |
34 |
use open qw( :std :utf8 ); |
|
|
35 |
binmode( STDOUT, ":utf8" ); |
34 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
36 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
35 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
37 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
|
|
38 |
|
39 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); |
36 |
my $cleanisbn = 1; |
40 |
my $cleanisbn = 1; |
37 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
41 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
38 |
my $framework = ''; |
42 |
my $framework = ''; |
Lines 46-52
GetOptions(
Link Here
|
46 |
'o|offset:f' => \$offset, |
50 |
'o|offset:f' => \$offset, |
47 |
'h' => \$version, |
51 |
'h' => \$version, |
48 |
'd' => \$delete, |
52 |
'd' => \$delete, |
49 |
't' => \$test_parameter, |
53 |
't|test' => \$test_parameter, |
50 |
's' => \$skip_marc8_conversion, |
54 |
's' => \$skip_marc8_conversion, |
51 |
'c:s' => \$char_encoding, |
55 |
'c:s' => \$char_encoding, |
52 |
'v:s' => \$verbose, |
56 |
'v:s' => \$verbose, |
Lines 56-61
GetOptions(
Link Here
|
56 |
'k|keepids:s' => \$keepids, |
60 |
'k|keepids:s' => \$keepids, |
57 |
'b|biblios' => \$biblios, |
61 |
'b|biblios' => \$biblios, |
58 |
'a|authorities' => \$authorities, |
62 |
'a|authorities' => \$authorities, |
|
|
63 |
'authtypes:s' => \$authtypes, |
64 |
'filter=s@' => \$filters, |
65 |
'insert' => \$insert, |
66 |
'update' => \$update, |
67 |
'all' => \$all, |
59 |
'match=s@' => \$match, |
68 |
'match=s@' => \$match, |
60 |
'i|isbn' => \$isbn_check, |
69 |
'i|isbn' => \$isbn_check, |
61 |
'x:s' => \$sourcetag, |
70 |
'x:s' => \$sourcetag, |
Lines 64-77
GetOptions(
Link Here
|
64 |
'cleanisbn!' => \$cleanisbn, |
73 |
'cleanisbn!' => \$cleanisbn, |
65 |
'dedupbarcode' => \$dedup_barcode, |
74 |
'dedupbarcode' => \$dedup_barcode, |
66 |
'framework=s' => \$framework, |
75 |
'framework=s' => \$framework, |
|
|
76 |
'yaml:s' => \$yamlfile, |
67 |
); |
77 |
); |
68 |
$biblios=!$authorities||$biblios; |
78 |
$biblios ||= !$authorities; |
|
|
79 |
$insert ||= !$update; |
80 |
|
81 |
if ($all) { |
82 |
$insert = 1; |
83 |
$update = 1; |
84 |
} |
69 |
|
85 |
|
70 |
if ($version || ($input_marc_file eq '')) { |
86 |
if ($version || ($input_marc_file eq '')) { |
71 |
pod2usage( -verbose => 2 ); |
87 |
pod2usage( -verbose => 2 ); |
72 |
exit; |
88 |
exit; |
73 |
} |
89 |
} |
74 |
|
90 |
|
|
|
91 |
my $heading_fields=get_heading_fields(); |
92 |
|
75 |
if (defined $idmapfl) { |
93 |
if (defined $idmapfl) { |
76 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
94 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
77 |
} |
95 |
} |
Lines 142-148
$batch->warnings_off();
Link Here
|
142 |
$batch->strict_off(); |
160 |
$batch->strict_off(); |
143 |
my $i=0; |
161 |
my $i=0; |
144 |
my $commitnum = $commit ? $commit : 50; |
162 |
my $commitnum = $commit ? $commit : 50; |
145 |
|
163 |
my $yamlhash; |
146 |
|
164 |
|
147 |
# Skip file offset |
165 |
# Skip file offset |
148 |
if ( $offset ) { |
166 |
if ( $offset ) { |
Lines 199-204
RECORD: while ( ) {
Link Here
|
199 |
next RECORD; |
217 |
next RECORD; |
200 |
} |
218 |
} |
201 |
} |
219 |
} |
|
|
220 |
SetUTF8Flag($record); |
202 |
my $isbn; |
221 |
my $isbn; |
203 |
# remove trailing - in isbn (only for biblios, of course) |
222 |
# remove trailing - in isbn (only for biblios, of course) |
204 |
if ($biblios && $cleanisbn) { |
223 |
if ($biblios && $cleanisbn) { |
Lines 212-256
RECORD: while ( ) {
Link Here
|
212 |
} |
231 |
} |
213 |
my $id; |
232 |
my $id; |
214 |
# search for duplicates (based on Local-number) |
233 |
# search for duplicates (based on Local-number) |
215 |
if ($match){ |
234 |
my $originalid; |
216 |
require C4::Search; |
235 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
217 |
my $query=build_query($match,$record); |
236 |
if ($match) { |
218 |
my $server=($authorities?'authorityserver':'biblioserver'); |
237 |
require C4::Search; |
219 |
my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
238 |
my $query = build_query( $match, $record ); |
220 |
die "unable to search the database for duplicates : $error" if (defined $error); |
239 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
221 |
#warn "$query $server : $totalhits"; |
240 |
$debug && warn $query; |
222 |
if ( @{$results} == 1 ){ |
241 |
my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
223 |
my $marcrecord = MARC::File::USMARC::decode($results->[0]); |
242 |
die "unable to search the database for duplicates : $error" if ( defined $error ); |
224 |
$id=GetRecordId($marcrecord,$tagid,$subfieldid); |
243 |
$debug && warn "$query $server : $totalhits"; |
225 |
} |
244 |
if ( $results && scalar(@$results) == 1 ) { |
226 |
elsif ( @{$results} > 1){ |
245 |
my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); |
227 |
$debug && warn "more than one match for $query"; |
246 |
SetUTF8Flag($marcrecord); |
228 |
} |
247 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
229 |
else { |
248 |
if ( $authorities && $marcFlavour ) { |
230 |
$debug && warn "nomatch for $query"; |
249 |
#Skip if authority in database is the same as the on in database |
231 |
} |
250 |
if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { |
|
|
251 |
if ($yamlfile) { |
252 |
$yamlhash->{$originalid}->{'authid'} = $id; |
253 |
|
254 |
# On récupère tous les souschamps des champs vedettes d'autorités |
255 |
my @subfields; |
256 |
foreach my $field ( $marcrecord->field("2..") ) { |
257 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
258 |
} |
259 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
260 |
} |
261 |
next; |
262 |
} |
263 |
} |
264 |
} elsif ( $results && scalar(@$results) > 1 ) { |
265 |
$debug && warn "more than one match for $query"; |
266 |
} else { |
267 |
$debug && warn "nomatch for $query"; |
268 |
} |
232 |
} |
269 |
} |
233 |
my $originalid; |
270 |
if ($keepids && $originalid) { |
234 |
if ($keepids){ |
271 |
my $storeidfield; |
235 |
$originalid=GetRecordId($record,$tagid,$subfieldid); |
272 |
if ( length($keepids) == 3 ) { |
236 |
if ($originalid){ |
273 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
237 |
my $storeidfield; |
274 |
} else { |
238 |
if (length($keepids)==3){ |
275 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
239 |
$storeidfield=MARC::Field->new($keepids,$originalid); |
276 |
} |
240 |
} |
277 |
$record->insert_fields_ordered($storeidfield); |
241 |
else { |
278 |
$record->delete_field( $record->field($tagid) ); |
242 |
$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); |
|
|
243 |
} |
244 |
$record->insert_fields_ordered($storeidfield); |
245 |
$record->delete_field($record->field($tagid)); |
246 |
} |
247 |
} |
279 |
} |
|
|
280 |
foreach my $stringfilter (@$filters) { |
281 |
if ( length($stringfilter) == 3 ) { |
282 |
foreach my $field ( $record->field($stringfilter) ) { |
283 |
$record->delete_field($field); |
284 |
$debug && warn "removed : ", $field->as_string; |
285 |
} |
286 |
} else { |
287 |
my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) |
288 |
if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; |
289 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
290 |
foreach my $field ( $record->field($removetag) ) { |
291 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
292 |
$debug && warn "Potentially removed : ", $field->subfield($removesubfield); |
293 |
} |
294 |
} |
295 |
} |
296 |
} |
297 |
|
248 |
unless ($test_parameter) { |
298 |
unless ($test_parameter) { |
249 |
if ($authorities){ |
299 |
if ($authorities){ |
250 |
use C4::AuthoritiesMarc; |
300 |
use C4::AuthoritiesMarc; |
251 |
my $authtypecode=GuessAuthTypeCode($record); |
301 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
252 |
my $authid= ($id?$id:GuessAuthId($record)); |
302 |
my $authid= ($id?$id:GuessAuthId($record)); |
253 |
if ($authid && GetAuthority($authid)){ |
303 |
if ($authid && GetAuthority($authid) && $update ){ |
254 |
## Authority has an id and is in database : Replace |
304 |
## Authority has an id and is in database : Replace |
255 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
305 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
256 |
if ($@){ |
306 |
if ($@){ |
Lines 283-288
RECORD: while ( ) {
Link Here
|
283 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
333 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
284 |
} |
334 |
} |
285 |
} |
335 |
} |
|
|
336 |
if ($yamlfile) { |
337 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
338 |
my @subfields; |
339 |
foreach my $field ( $record->field("2..") ) { |
340 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
341 |
} |
342 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
343 |
} |
286 |
} |
344 |
} |
287 |
else { |
345 |
else { |
288 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
346 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
Lines 306-325
RECORD: while ( ) {
Link Here
|
306 |
} |
364 |
} |
307 |
# create biblio, unless we already have it ( either match or isbn ) |
365 |
# create biblio, unless we already have it ( either match or isbn ) |
308 |
if ($biblionumber) { |
366 |
if ($biblionumber) { |
309 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} |
367 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; |
310 |
} |
368 |
if ($update) { |
311 |
else |
369 |
eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; |
312 |
{ |
370 |
if ($@) { |
313 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; |
371 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
|
|
372 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
373 |
next RECORD; |
374 |
} else { |
375 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
376 |
} |
377 |
} else { |
378 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
379 |
} |
380 |
} else { |
381 |
if ($insert) { |
382 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; |
383 |
if ($@) { |
384 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
385 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
386 |
next RECORD; |
387 |
} else { |
388 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
389 |
} |
390 |
} else { |
391 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
392 |
} |
314 |
} |
393 |
} |
315 |
if ( $@ ) { |
|
|
316 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
317 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); |
318 |
next RECORD; |
319 |
} |
320 |
else{ |
321 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); |
322 |
} |
323 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
394 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
324 |
my $error_adding = $@; |
395 |
my $error_adding = $@; |
325 |
# Work on a clone so that if there are real errors, we can maybe |
396 |
# Work on a clone so that if there are real errors, we can maybe |
Lines 384-389
RECORD: while ( ) {
Link Here
|
384 |
if ($#{ $errors_ref } > -1) { |
455 |
if ($#{ $errors_ref } > -1) { |
385 |
report_item_errors($biblionumber, $errors_ref); |
456 |
report_item_errors($biblionumber, $errors_ref); |
386 |
} |
457 |
} |
|
|
458 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
387 |
} |
459 |
} |
388 |
$dbh->commit() if (0 == $i % $commitnum); |
460 |
$dbh->commit() if (0 == $i % $commitnum); |
389 |
} |
461 |
} |
Lines 407-412
if ($logfile){
Link Here
|
407 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
479 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
408 |
$loghandle->close; |
480 |
$loghandle->close; |
409 |
} |
481 |
} |
|
|
482 |
if ($yamlfile) { |
483 |
open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; |
484 |
print YAML Dump($yamlhash); |
485 |
} |
410 |
exit 0; |
486 |
exit 0; |
411 |
|
487 |
|
412 |
sub GetRecordId{ |
488 |
sub GetRecordId{ |
Lines 462-470
sub report_item_errors {
Link Here
|
462 |
} |
538 |
} |
463 |
sub printlog{ |
539 |
sub printlog{ |
464 |
my $logelements=shift; |
540 |
my $logelements=shift; |
465 |
print $loghandle join (";",@$logelements{qw<id op status>}),"\n"; |
541 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
466 |
} |
542 |
} |
|
|
543 |
sub get_heading_fields{ |
544 |
my $headingfields; |
545 |
if ($authtypes){ |
546 |
$headingfields=YAML::LoadFile($authtypes); |
467 |
|
547 |
|
|
|
548 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
549 |
$debug && warn YAML::Dump($headingfields); |
550 |
} |
551 |
unless ($headingfields){ |
552 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
553 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
554 |
} |
555 |
return $headingfields; |
556 |
} |
468 |
|
557 |
|
469 |
=head1 NAME |
558 |
=head1 NAME |
470 |
|
559 |
|
Lines 526-532
The I<NUMBER> of records to wait before performing a 'commit' operation
Link Here
|
526 |
|
615 |
|
527 |
File logs actions done for each record and their status into file |
616 |
File logs actions done for each record and their status into file |
528 |
|
617 |
|
529 |
=item B<-t> |
618 |
=item B<-t, -test> |
530 |
|
619 |
|
531 |
Test mode: parses the file, saying what he would do, but doing nothing. |
620 |
Test mode: parses the file, saying what he would do, but doing nothing. |
532 |
|
621 |
|
Lines 549-554
biblioitems, items
Link Here
|
549 |
|
638 |
|
550 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
639 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
551 |
|
640 |
|
|
|
641 |
=item B<-authtypes> |
642 |
|
643 |
file yamlfile with authoritiesTypes and distinguishable record field in order |
644 |
to store the correct authtype |
645 |
|
646 |
=item B<-yaml> |
647 |
|
648 |
yaml file format a yaml file with ids |
649 |
keepids field store ids in field (usefull for authorities, |
650 |
where 001 contains the authid for Koha, that can contain a very valuable |
651 |
info for authorities coming from LOC or BNF. useless for biblios probably |
652 |
|
653 |
=item B<-insert> |
654 |
|
655 |
if set, only insert when possible |
656 |
|
657 |
=item B<-update> |
658 |
|
659 |
if set, only updates (any biblio should have a matching record) |
660 |
|
661 |
=item B<-all> |
662 |
|
663 |
if set, do whatever is required |
664 |
|
552 |
=item B<-k, -keepids>=<FIELD> |
665 |
=item B<-k, -keepids>=<FIELD> |
553 |
|
666 |
|
554 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
667 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
555 |
- |
|
|