Lines 24-38
use C4::Koha;
Link Here
|
24 |
use C4::Debug; |
24 |
use C4::Debug; |
25 |
use C4::Charset; |
25 |
use C4::Charset; |
26 |
use C4::Items; |
26 |
use C4::Items; |
|
|
27 |
use YAML; |
27 |
use Unicode::Normalize; |
28 |
use Unicode::Normalize; |
28 |
use Time::HiRes qw(gettimeofday); |
29 |
use Time::HiRes qw(gettimeofday); |
29 |
use Getopt::Long; |
30 |
use Getopt::Long; |
30 |
use IO::File; |
31 |
use IO::File; |
31 |
use Pod::Usage; |
32 |
use Pod::Usage; |
32 |
|
33 |
|
33 |
binmode STDOUT, ':encoding(UTF-8)'; |
34 |
use open qw( :std :utf8 ); |
|
|
35 |
binmode( STDOUT, ":utf8" ); |
34 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
36 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
35 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
37 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
|
|
38 |
|
39 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); |
36 |
my $cleanisbn = 1; |
40 |
my $cleanisbn = 1; |
37 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
41 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
38 |
|
42 |
|
Lines 45-51
GetOptions(
Link Here
|
45 |
'o|offset:f' => \$offset, |
49 |
'o|offset:f' => \$offset, |
46 |
'h' => \$version, |
50 |
'h' => \$version, |
47 |
'd' => \$delete, |
51 |
'd' => \$delete, |
48 |
't' => \$test_parameter, |
52 |
't|test' => \$test_parameter, |
49 |
's' => \$skip_marc8_conversion, |
53 |
's' => \$skip_marc8_conversion, |
50 |
'c:s' => \$char_encoding, |
54 |
'c:s' => \$char_encoding, |
51 |
'v:s' => \$verbose, |
55 |
'v:s' => \$verbose, |
Lines 55-60
GetOptions(
Link Here
|
55 |
'k|keepids:s' => \$keepids, |
59 |
'k|keepids:s' => \$keepids, |
56 |
'b|biblios' => \$biblios, |
60 |
'b|biblios' => \$biblios, |
57 |
'a|authorities' => \$authorities, |
61 |
'a|authorities' => \$authorities, |
|
|
62 |
'authtypes:s' => \$authtypes, |
63 |
'filter=s@' => \$filters, |
64 |
'insert' => \$insert, |
65 |
'update' => \$update, |
66 |
'all' => \$all, |
58 |
'match=s@' => \$match, |
67 |
'match=s@' => \$match, |
59 |
'i|isbn' => \$isbn_check, |
68 |
'i|isbn' => \$isbn_check, |
60 |
'x:s' => \$sourcetag, |
69 |
'x:s' => \$sourcetag, |
Lines 62-75
GetOptions(
Link Here
|
62 |
'idmap:s' => \$idmapfl, |
71 |
'idmap:s' => \$idmapfl, |
63 |
'cleanisbn!' => \$cleanisbn, |
72 |
'cleanisbn!' => \$cleanisbn, |
64 |
'dedupbarcode' => \$dedup_barcode, |
73 |
'dedupbarcode' => \$dedup_barcode, |
|
|
74 |
'yaml:s' => \$yamlfile, |
65 |
); |
75 |
); |
66 |
$biblios=!$authorities||$biblios; |
76 |
$biblios ||= !$authorities; |
|
|
77 |
$insert ||= !$update; |
78 |
|
79 |
if ($all) { |
80 |
$insert = 1; |
81 |
$update = 1; |
82 |
} |
67 |
|
83 |
|
68 |
if ($version || ($input_marc_file eq '')) { |
84 |
if ($version || ($input_marc_file eq '')) { |
69 |
pod2usage( -verbose => 2 ); |
85 |
pod2usage( -verbose => 2 ); |
70 |
exit; |
86 |
exit; |
71 |
} |
87 |
} |
72 |
|
88 |
|
|
|
89 |
my $heading_fields=get_heading_fields(); |
90 |
|
73 |
if (defined $idmapfl) { |
91 |
if (defined $idmapfl) { |
74 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
92 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
75 |
} |
93 |
} |
Lines 140-146
$batch->warnings_off();
Link Here
|
140 |
$batch->strict_off(); |
158 |
$batch->strict_off(); |
141 |
my $i=0; |
159 |
my $i=0; |
142 |
my $commitnum = $commit ? $commit : 50; |
160 |
my $commitnum = $commit ? $commit : 50; |
143 |
|
161 |
my $yamlhash; |
144 |
|
162 |
|
145 |
# Skip file offset |
163 |
# Skip file offset |
146 |
if ( $offset ) { |
164 |
if ( $offset ) { |
Lines 197-202
RECORD: while ( ) {
Link Here
|
197 |
next RECORD; |
215 |
next RECORD; |
198 |
} |
216 |
} |
199 |
} |
217 |
} |
|
|
218 |
SetUTF8Flag($record); |
200 |
my $isbn; |
219 |
my $isbn; |
201 |
# remove trailing - in isbn (only for biblios, of course) |
220 |
# remove trailing - in isbn (only for biblios, of course) |
202 |
if ($biblios && $cleanisbn) { |
221 |
if ($biblios && $cleanisbn) { |
Lines 210-254
RECORD: while ( ) {
Link Here
|
210 |
} |
229 |
} |
211 |
my $id; |
230 |
my $id; |
212 |
# search for duplicates (based on Local-number) |
231 |
# search for duplicates (based on Local-number) |
213 |
if ($match){ |
232 |
my $originalid; |
214 |
require C4::Search; |
233 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
215 |
my $query=build_query($match,$record); |
234 |
if ($match) { |
216 |
my $server=($authorities?'authorityserver':'biblioserver'); |
235 |
require C4::Search; |
217 |
my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
236 |
my $query = build_query( $match, $record ); |
218 |
die "unable to search the database for duplicates : $error" if (defined $error); |
237 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
219 |
#warn "$query $server : $totalhits"; |
238 |
$debug && warn $query; |
220 |
if ( @{$results} == 1 ){ |
239 |
my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
221 |
my $marcrecord = MARC::File::USMARC::decode($results->[0]); |
240 |
die "unable to search the database for duplicates : $error" if ( defined $error ); |
222 |
$id=GetRecordId($marcrecord,$tagid,$subfieldid); |
241 |
$debug && warn "$query $server : $totalhits"; |
223 |
} |
242 |
if ( $results && scalar(@$results) == 1 ) { |
224 |
elsif ( @{$results} > 1){ |
243 |
my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); |
225 |
$debug && warn "more than one match for $query"; |
244 |
SetUTF8Flag($marcrecord); |
226 |
} |
245 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
227 |
else { |
246 |
if ( $authorities && $marcFlavour ) { |
228 |
$debug && warn "nomatch for $query"; |
247 |
#Skip if authority in database is the same as the on in database |
229 |
} |
248 |
if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { |
|
|
249 |
if ($yamlfile) { |
250 |
$yamlhash->{$originalid}->{'authid'} = $id; |
251 |
|
252 |
# On récupère tous les souschamps des champs vedettes d'autorités |
253 |
my @subfields; |
254 |
foreach my $field ( $marcrecord->field("2..") ) { |
255 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
256 |
} |
257 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
258 |
} |
259 |
next; |
260 |
} |
261 |
} |
262 |
} elsif ( $results && scalar(@$results) > 1 ) { |
263 |
$debug && warn "more than one match for $query"; |
264 |
} else { |
265 |
$debug && warn "nomatch for $query"; |
266 |
} |
230 |
} |
267 |
} |
231 |
my $originalid; |
268 |
if ($keepids && $originalid) { |
232 |
if ($keepids){ |
269 |
my $storeidfield; |
233 |
$originalid=GetRecordId($record,$tagid,$subfieldid); |
270 |
if ( length($keepids) == 3 ) { |
234 |
if ($originalid){ |
271 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
235 |
my $storeidfield; |
272 |
} else { |
236 |
if (length($keepids)==3){ |
273 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
237 |
$storeidfield=MARC::Field->new($keepids,$originalid); |
274 |
} |
238 |
} |
275 |
$record->insert_fields_ordered($storeidfield); |
239 |
else { |
276 |
$record->delete_field( $record->field($tagid) ); |
240 |
$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); |
|
|
241 |
} |
242 |
$record->insert_fields_ordered($storeidfield); |
243 |
$record->delete_field($record->field($tagid)); |
244 |
} |
245 |
} |
277 |
} |
|
|
278 |
foreach my $stringfilter (@$filters) { |
279 |
if ( length($stringfilter) == 3 ) { |
280 |
foreach my $field ( $record->field($stringfilter) ) { |
281 |
$record->delete_field($field); |
282 |
$debug && warn "removed : ", $field->as_string; |
283 |
} |
284 |
} else { |
285 |
my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) |
286 |
if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; |
287 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
288 |
foreach my $field ( $record->field($removetag) ) { |
289 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
290 |
$debug && warn "Potentially removed : ", $field->subfield($removesubfield); |
291 |
} |
292 |
} |
293 |
} |
294 |
} |
295 |
|
246 |
unless ($test_parameter) { |
296 |
unless ($test_parameter) { |
247 |
if ($authorities){ |
297 |
if ($authorities){ |
248 |
use C4::AuthoritiesMarc; |
298 |
use C4::AuthoritiesMarc; |
249 |
my $authtypecode=GuessAuthTypeCode($record); |
299 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
250 |
my $authid= ($id?$id:GuessAuthId($record)); |
300 |
my $authid= ($id?$id:GuessAuthId($record)); |
251 |
if ($authid && GetAuthority($authid)){ |
301 |
if ($authid && GetAuthority($authid) && $update ){ |
252 |
## Authority has an id and is in database : Replace |
302 |
## Authority has an id and is in database : Replace |
253 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
303 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
254 |
if ($@){ |
304 |
if ($@){ |
Lines 281-286
RECORD: while ( ) {
Link Here
|
281 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
331 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
282 |
} |
332 |
} |
283 |
} |
333 |
} |
|
|
334 |
if ($yamlfile) { |
335 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
336 |
my @subfields; |
337 |
foreach my $field ( $record->field("2..") ) { |
338 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
339 |
} |
340 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
341 |
} |
284 |
} |
342 |
} |
285 |
else { |
343 |
else { |
286 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
344 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
Lines 304-323
RECORD: while ( ) {
Link Here
|
304 |
} |
362 |
} |
305 |
# create biblio, unless we already have it ( either match or isbn ) |
363 |
# create biblio, unless we already have it ( either match or isbn ) |
306 |
if ($biblionumber) { |
364 |
if ($biblionumber) { |
307 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} |
365 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; |
308 |
} |
366 |
if ($update) { |
309 |
else |
367 |
eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; |
310 |
{ |
368 |
if ($@) { |
311 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) }; |
369 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
|
|
370 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
371 |
next RECORD; |
372 |
} else { |
373 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
374 |
} |
375 |
} else { |
376 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
377 |
} |
378 |
} else { |
379 |
if ($insert) { |
380 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; |
381 |
if ($@) { |
382 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
383 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
384 |
next RECORD; |
385 |
} else { |
386 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
387 |
} |
388 |
} else { |
389 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
390 |
} |
312 |
} |
391 |
} |
313 |
if ( $@ ) { |
|
|
314 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
315 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); |
316 |
next RECORD; |
317 |
} |
318 |
else{ |
319 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); |
320 |
} |
321 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
392 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
322 |
my $error_adding = $@; |
393 |
my $error_adding = $@; |
323 |
# Work on a clone so that if there are real errors, we can maybe |
394 |
# Work on a clone so that if there are real errors, we can maybe |
Lines 382-387
RECORD: while ( ) {
Link Here
|
382 |
if ($#{ $errors_ref } > -1) { |
453 |
if ($#{ $errors_ref } > -1) { |
383 |
report_item_errors($biblionumber, $errors_ref); |
454 |
report_item_errors($biblionumber, $errors_ref); |
384 |
} |
455 |
} |
|
|
456 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
385 |
} |
457 |
} |
386 |
$dbh->commit() if (0 == $i % $commitnum); |
458 |
$dbh->commit() if (0 == $i % $commitnum); |
387 |
} |
459 |
} |
Lines 405-410
if ($logfile){
Link Here
|
405 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
477 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
406 |
$loghandle->close; |
478 |
$loghandle->close; |
407 |
} |
479 |
} |
|
|
480 |
if ($yamlfile) { |
481 |
open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; |
482 |
print YAML Dump($yamlhash); |
483 |
} |
408 |
exit 0; |
484 |
exit 0; |
409 |
|
485 |
|
410 |
sub GetRecordId{ |
486 |
sub GetRecordId{ |
Lines 460-468
sub report_item_errors {
Link Here
|
460 |
} |
536 |
} |
461 |
sub printlog{ |
537 |
sub printlog{ |
462 |
my $logelements=shift; |
538 |
my $logelements=shift; |
463 |
print $loghandle join (";",@$logelements{qw<id op status>}),"\n"; |
539 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
464 |
} |
540 |
} |
|
|
541 |
sub get_heading_fields{ |
542 |
my $headingfields; |
543 |
if ($authtypes){ |
544 |
$headingfields=YAML::LoadFile($authtypes); |
465 |
|
545 |
|
|
|
546 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
547 |
$debug && warn YAML::Dump($headingfields); |
548 |
} |
549 |
unless ($headingfields){ |
550 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
551 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
552 |
} |
553 |
return $headingfields; |
554 |
} |
466 |
|
555 |
|
467 |
=head1 NAME |
556 |
=head1 NAME |
468 |
|
557 |
|
Lines 524-530
The I<NUMBER> of records to wait before performing a 'commit' operation
Link Here
|
524 |
|
613 |
|
525 |
File logs actions done for each record and their status into file |
614 |
File logs actions done for each record and their status into file |
526 |
|
615 |
|
527 |
=item B<-t> |
616 |
=item B<-t, -test> |
528 |
|
617 |
|
529 |
Test mode: parses the file, saying what he would do, but doing nothing. |
618 |
Test mode: parses the file, saying what he would do, but doing nothing. |
530 |
|
619 |
|
Lines 547-552
biblioitems, items
Link Here
|
547 |
|
636 |
|
548 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
637 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
549 |
|
638 |
|
|
|
639 |
=item B<-authtypes> |
640 |
|
641 |
file yamlfile with authoritiesTypes and distinguishable record field in order |
642 |
to store the correct authtype |
643 |
|
644 |
=item B<-yaml> |
645 |
|
646 |
yaml file format a yaml file with ids |
647 |
keepids field store ids in field (usefull for authorities, |
648 |
where 001 contains the authid for Koha, that can contain a very valuable |
649 |
info for authorities coming from LOC or BNF. useless for biblios probably |
650 |
|
651 |
=item B<-insert> |
652 |
|
653 |
if set, only insert when possible |
654 |
|
655 |
=item B<-update> |
656 |
|
657 |
if set, only updates (any biblio should have a matching record) |
658 |
|
659 |
=item B<-all> |
660 |
|
661 |
if set, do whatever is required |
662 |
|
550 |
=item B<-k, -keepids>=<FIELD> |
663 |
=item B<-k, -keepids>=<FIELD> |
551 |
|
664 |
|
552 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
665 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
553 |
- |
|
|