Lines 24-39
use C4::Koha;
Link Here
|
24 |
use C4::Debug; |
24 |
use C4::Debug; |
25 |
use C4::Charset; |
25 |
use C4::Charset; |
26 |
use C4::Items; |
26 |
use C4::Items; |
|
|
27 |
use YAML; |
27 |
use Unicode::Normalize; |
28 |
use Unicode::Normalize; |
28 |
use Time::HiRes qw(gettimeofday); |
29 |
use Time::HiRes qw(gettimeofday); |
29 |
use Getopt::Long; |
30 |
use Getopt::Long; |
30 |
use IO::File; |
31 |
use IO::File; |
31 |
use Pod::Usage; |
32 |
use Pod::Usage; |
32 |
|
33 |
|
33 |
binmode STDOUT, ':encoding(UTF-8)'; |
34 |
use open qw( :std :utf8 ); |
|
|
35 |
binmode( STDOUT, ":utf8" ); |
34 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
36 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
35 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
37 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
36 |
my ($sourcetag,$sourcesubfield,$idmapfl); |
38 |
my ($sourcetag,$sourcesubfield,$idmapfl); |
|
|
39 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); |
37 |
my $cleanisbn = 1; |
40 |
my $cleanisbn = 1; |
38 |
|
41 |
|
39 |
$|=1; |
42 |
$|=1; |
Lines 45-51
GetOptions(
Link Here
|
45 |
'o|offset:f' => \$offset, |
48 |
'o|offset:f' => \$offset, |
46 |
'h' => \$version, |
49 |
'h' => \$version, |
47 |
'd' => \$delete, |
50 |
'd' => \$delete, |
48 |
't' => \$test_parameter, |
51 |
't|test' => \$test_parameter, |
49 |
's' => \$skip_marc8_conversion, |
52 |
's' => \$skip_marc8_conversion, |
50 |
'c:s' => \$char_encoding, |
53 |
'c:s' => \$char_encoding, |
51 |
'v:s' => \$verbose, |
54 |
'v:s' => \$verbose, |
Lines 55-74
GetOptions(
Link Here
|
55 |
'k|keepids:s' => \$keepids, |
58 |
'k|keepids:s' => \$keepids, |
56 |
'b|biblios' => \$biblios, |
59 |
'b|biblios' => \$biblios, |
57 |
'a|authorities' => \$authorities, |
60 |
'a|authorities' => \$authorities, |
|
|
61 |
'authtypes:s' => \$authtypes, |
62 |
'filter=s@' => \$filters, |
63 |
'insert' => \$insert, |
64 |
'update' => \$update, |
65 |
'all' => \$all, |
58 |
'match=s@' => \$match, |
66 |
'match=s@' => \$match, |
59 |
'i|isbn' => \$isbn_check, |
67 |
'i|isbn' => \$isbn_check, |
60 |
'x:s' => \$sourcetag, |
68 |
'x:s' => \$sourcetag, |
61 |
'y:s' => \$sourcesubfield, |
69 |
'y:s' => \$sourcesubfield, |
62 |
'idmap:s' => \$idmapfl, |
70 |
'idmap:s' => \$idmapfl, |
63 |
'cleanisbn!' => \$cleanisbn, |
71 |
'cleanisbn!' => \$cleanisbn, |
|
|
72 |
'yaml:s' => \$yamlfile, |
64 |
); |
73 |
); |
65 |
$biblios=!$authorities||$biblios; |
74 |
$biblios ||= !$authorities; |
|
|
75 |
$insert ||= !$update; |
76 |
|
77 |
if ($all) { |
78 |
$insert = 1; |
79 |
$update = 1; |
80 |
} |
66 |
|
81 |
|
67 |
if ($version || ($input_marc_file eq '')) { |
82 |
if ($version || ($input_marc_file eq '')) { |
68 |
pod2usage( -verbose => 2 ); |
83 |
pod2usage( -verbose => 2 ); |
69 |
exit; |
84 |
exit; |
70 |
} |
85 |
} |
71 |
|
86 |
|
|
|
87 |
my $heading_fields=get_heading_fields(); |
88 |
|
72 |
if (defined $idmapfl) { |
89 |
if (defined $idmapfl) { |
73 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
90 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
74 |
} |
91 |
} |
Lines 139-145
$batch->warnings_off();
Link Here
|
139 |
$batch->strict_off(); |
156 |
$batch->strict_off(); |
140 |
my $i=0; |
157 |
my $i=0; |
141 |
my $commitnum = $commit ? $commit : 50; |
158 |
my $commitnum = $commit ? $commit : 50; |
142 |
|
159 |
my $yamlhash; |
143 |
|
160 |
|
144 |
# Skip file offset |
161 |
# Skip file offset |
145 |
if ( $offset ) { |
162 |
if ( $offset ) { |
Lines 196-201
RECORD: while ( ) {
Link Here
|
196 |
next RECORD; |
213 |
next RECORD; |
197 |
} |
214 |
} |
198 |
} |
215 |
} |
|
|
216 |
SetUTF8Flag($record); |
199 |
my $isbn; |
217 |
my $isbn; |
200 |
# remove trailing - in isbn (only for biblios, of course) |
218 |
# remove trailing - in isbn (only for biblios, of course) |
201 |
if ($biblios && $cleanisbn) { |
219 |
if ($biblios && $cleanisbn) { |
Lines 209-253
RECORD: while ( ) {
Link Here
|
209 |
} |
227 |
} |
210 |
my $id; |
228 |
my $id; |
211 |
# search for duplicates (based on Local-number) |
229 |
# search for duplicates (based on Local-number) |
212 |
if ($match){ |
230 |
my $originalid; |
213 |
require C4::Search; |
231 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
214 |
my $query=build_query($match,$record); |
232 |
if ($match) { |
215 |
my $server=($authorities?'authorityserver':'biblioserver'); |
233 |
require C4::Search; |
216 |
my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
234 |
my $query = build_query( $match, $record ); |
217 |
die "unable to search the database for duplicates : $error" if (defined $error); |
235 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
218 |
#warn "$query $server : $totalhits"; |
236 |
$debug && warn $query; |
219 |
if ( @{$results} == 1 ){ |
237 |
my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
220 |
my $marcrecord = MARC::File::USMARC::decode($results->[0]); |
238 |
die "unable to search the database for duplicates : $error" if ( defined $error ); |
221 |
$id=GetRecordId($marcrecord,$tagid,$subfieldid); |
239 |
$debug && warn "$query $server : $totalhits"; |
222 |
} |
240 |
if ( $results && scalar(@$results) == 1 ) { |
223 |
elsif ( @{$results} > 1){ |
241 |
my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); |
224 |
$debug && warn "more than one match for $query"; |
242 |
SetUTF8Flag($marcrecord); |
225 |
} |
243 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
226 |
else { |
244 |
if ( $authorities && $marcFlavour ) { |
227 |
$debug && warn "nomatch for $query"; |
245 |
#Skip if authority in database is the same as the on in database |
228 |
} |
246 |
if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { |
|
|
247 |
if ($yamlfile) { |
248 |
$yamlhash->{$originalid}->{'authid'} = $id; |
249 |
|
250 |
# On récupère tous les souschamps des champs vedettes d'autorités |
251 |
my @subfields; |
252 |
foreach my $field ( $marcrecord->field("2..") ) { |
253 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
254 |
} |
255 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
256 |
} |
257 |
next; |
258 |
} |
259 |
} |
260 |
} elsif ( $results && scalar(@$results) > 1 ) { |
261 |
$debug && warn "more than one match for $query"; |
262 |
} else { |
263 |
$debug && warn "nomatch for $query"; |
264 |
} |
229 |
} |
265 |
} |
230 |
my $originalid; |
266 |
if ($keepids && $originalid) { |
231 |
if ($keepids){ |
267 |
my $storeidfield; |
232 |
$originalid=GetRecordId($record,$tagid,$subfieldid); |
268 |
if ( length($keepids) == 3 ) { |
233 |
if ($originalid){ |
269 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
234 |
my $storeidfield; |
270 |
} else { |
235 |
if (length($keepids)==3){ |
271 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
236 |
$storeidfield=MARC::Field->new($keepids,$originalid); |
272 |
} |
237 |
} |
273 |
$record->insert_fields_ordered($storeidfield); |
238 |
else { |
274 |
$record->delete_field( $record->field($tagid) ); |
239 |
$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); |
275 |
} |
240 |
} |
276 |
foreach my $stringfilter (@$filters) { |
241 |
$record->insert_fields_ordered($storeidfield); |
277 |
if ( length($stringfilter) == 3 ) { |
242 |
$record->delete_field($record->field($tagid)); |
278 |
foreach my $field ( $record->field($stringfilter) ) { |
243 |
} |
279 |
$record->delete_field($field); |
|
|
280 |
$debug && warn "removed : ", $field->as_string; |
281 |
} |
282 |
} else { |
283 |
my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) |
284 |
if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; |
285 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
286 |
foreach my $field ( $record->field($removetag) ) { |
287 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
288 |
$debug && warn "Potentially removed : ", $field->subfield($removesubfield); |
289 |
} |
290 |
} |
291 |
} |
244 |
} |
292 |
} |
|
|
293 |
|
245 |
unless ($test_parameter) { |
294 |
unless ($test_parameter) { |
246 |
if ($authorities){ |
295 |
if ($authorities){ |
247 |
use C4::AuthoritiesMarc; |
296 |
use C4::AuthoritiesMarc; |
248 |
my $authtypecode=GuessAuthTypeCode($record); |
297 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
249 |
my $authid= ($id?$id:GuessAuthId($record)); |
298 |
my $authid= ($id?$id:GuessAuthId($record)); |
250 |
if ($authid && GetAuthority($authid)){ |
299 |
if ($authid && GetAuthority($authid) && $update ){ |
251 |
## Authority has an id and is in database : Replace |
300 |
## Authority has an id and is in database : Replace |
252 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
301 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
253 |
if ($@){ |
302 |
if ($@){ |
Lines 280-285
RECORD: while ( ) {
Link Here
|
280 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
329 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
281 |
} |
330 |
} |
282 |
} |
331 |
} |
|
|
332 |
if ($yamlfile) { |
333 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
334 |
my @subfields; |
335 |
foreach my $field ( $record->field("2..") ) { |
336 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
337 |
} |
338 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
339 |
} |
283 |
} |
340 |
} |
284 |
else { |
341 |
else { |
285 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
342 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
Lines 303-322
RECORD: while ( ) {
Link Here
|
303 |
} |
360 |
} |
304 |
# create biblio, unless we already have it ( either match or isbn ) |
361 |
# create biblio, unless we already have it ( either match or isbn ) |
305 |
if ($biblionumber) { |
362 |
if ($biblionumber) { |
306 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} |
363 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; |
307 |
} |
364 |
if ($update) { |
308 |
else |
365 |
eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; |
309 |
{ |
366 |
if ($@) { |
310 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) }; |
367 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
|
|
368 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
369 |
next RECORD; |
370 |
} else { |
371 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
372 |
} |
373 |
} else { |
374 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
375 |
} |
376 |
} else { |
377 |
if ($insert) { |
378 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; |
379 |
if ($@) { |
380 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
381 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
382 |
next RECORD; |
383 |
} else { |
384 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
385 |
} |
386 |
} else { |
387 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
388 |
} |
311 |
} |
389 |
} |
312 |
if ( $@ ) { |
|
|
313 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
314 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); |
315 |
next RECORD; |
316 |
} |
317 |
else{ |
318 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); |
319 |
} |
320 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
390 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
321 |
if ( $@ ) { |
391 |
if ( $@ ) { |
322 |
warn "ERROR: Adding items to bib $biblionumber failed: $@\n"; |
392 |
warn "ERROR: Adding items to bib $biblionumber failed: $@\n"; |
Lines 335-340
RECORD: while ( ) {
Link Here
|
335 |
if ($#{ $errors_ref } > -1) { |
405 |
if ($#{ $errors_ref } > -1) { |
336 |
report_item_errors($biblionumber, $errors_ref); |
406 |
report_item_errors($biblionumber, $errors_ref); |
337 |
} |
407 |
} |
|
|
408 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
338 |
} |
409 |
} |
339 |
$dbh->commit() if (0 == $i % $commitnum); |
410 |
$dbh->commit() if (0 == $i % $commitnum); |
340 |
} |
411 |
} |
Lines 358-363
if ($logfile){
Link Here
|
358 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
429 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
359 |
$loghandle->close; |
430 |
$loghandle->close; |
360 |
} |
431 |
} |
|
|
432 |
if ($yamlfile) { |
433 |
open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; |
434 |
print YAML Dump($yamlhash); |
435 |
} |
361 |
exit 0; |
436 |
exit 0; |
362 |
|
437 |
|
363 |
sub GetRecordId{ |
438 |
sub GetRecordId{ |
Lines 412-420
sub report_item_errors {
Link Here
|
412 |
} |
487 |
} |
413 |
sub printlog{ |
488 |
sub printlog{ |
414 |
my $logelements=shift; |
489 |
my $logelements=shift; |
415 |
print $loghandle join (";",@$logelements{qw<id op status>}),"\n"; |
490 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
416 |
} |
491 |
} |
|
|
492 |
sub get_heading_fields{ |
493 |
my $headingfields; |
494 |
if ($authtypes){ |
495 |
$headingfields=YAML::LoadFile($authtypes); |
417 |
|
496 |
|
|
|
497 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
498 |
$debug && warn YAML::Dump($headingfields); |
499 |
} |
500 |
unless ($headingfields){ |
501 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
502 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
503 |
} |
504 |
return $headingfields; |
505 |
} |
418 |
|
506 |
|
419 |
=head1 NAME |
507 |
=head1 NAME |
420 |
|
508 |
|
Lines 476-482
The I<NUMBER> of records to wait before performing a 'commit' operation
Link Here
|
476 |
|
564 |
|
477 |
File logs actions done for each record and their status into file |
565 |
File logs actions done for each record and their status into file |
478 |
|
566 |
|
479 |
=item B<-t> |
567 |
=item B<-t, -test> |
480 |
|
568 |
|
481 |
Test mode: parses the file, saying what he would do, but doing nothing. |
569 |
Test mode: parses the file, saying what he would do, but doing nothing. |
482 |
|
570 |
|
Lines 499-504
biblioitems, items
Link Here
|
499 |
|
587 |
|
500 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
588 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
501 |
|
589 |
|
|
|
590 |
=item B<-authtypes> |
591 |
|
592 |
file yamlfile with authoritiesTypes and distinguishable record field in order |
593 |
to store the correct authtype |
594 |
|
595 |
=item B<-yaml> |
596 |
|
597 |
yaml file format a yaml file with ids |
598 |
keepids field store ids in field (usefull for authorities, |
599 |
where 001 contains the authid for Koha, that can contain a very valuable |
600 |
info for authorities coming from LOC or BNF. useless for biblios probably |
601 |
|
602 |
=item B<-insert> |
603 |
|
604 |
if set, only insert when possible |
605 |
|
606 |
=item B<-update> |
607 |
|
608 |
if set, only updates (any biblio should have a matching record) |
609 |
|
610 |
=item B<-all> |
611 |
|
612 |
if set, do whatever is required |
613 |
|
502 |
=item B<-k, -keepids>=<FIELD> |
614 |
=item B<-k, -keepids>=<FIELD> |
503 |
|
615 |
|
504 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
616 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
505 |
- |
|
|