Lines 24-38
use C4::Koha;
Link Here
|
24 |
use C4::Debug; |
24 |
use C4::Debug; |
25 |
use C4::Charset; |
25 |
use C4::Charset; |
26 |
use C4::Items; |
26 |
use C4::Items; |
|
|
27 |
use YAML; |
27 |
use Unicode::Normalize; |
28 |
use Unicode::Normalize; |
28 |
use Time::HiRes qw(gettimeofday); |
29 |
use Time::HiRes qw(gettimeofday); |
29 |
use Getopt::Long; |
30 |
use Getopt::Long; |
30 |
use IO::File; |
31 |
use IO::File; |
31 |
use Pod::Usage; |
32 |
use Pod::Usage; |
32 |
|
33 |
|
33 |
binmode STDOUT, ':encoding(UTF-8)'; |
34 |
use open qw( :std :utf8 ); |
|
|
35 |
binmode( STDOUT, ":utf8" ); |
34 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
36 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
35 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
37 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
|
|
38 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); |
36 |
my $cleanisbn = 1; |
39 |
my $cleanisbn = 1; |
37 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
40 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
38 |
my $framework = ''; |
41 |
my $framework = ''; |
Lines 46-52
GetOptions(
Link Here
|
46 |
'o|offset:f' => \$offset, |
49 |
'o|offset:f' => \$offset, |
47 |
'h' => \$version, |
50 |
'h' => \$version, |
48 |
'd' => \$delete, |
51 |
'd' => \$delete, |
49 |
't' => \$test_parameter, |
52 |
't|test' => \$test_parameter, |
50 |
's' => \$skip_marc8_conversion, |
53 |
's' => \$skip_marc8_conversion, |
51 |
'c:s' => \$char_encoding, |
54 |
'c:s' => \$char_encoding, |
52 |
'v:s' => \$verbose, |
55 |
'v:s' => \$verbose, |
Lines 56-77
GetOptions(
Link Here
|
56 |
'k|keepids:s' => \$keepids, |
59 |
'k|keepids:s' => \$keepids, |
57 |
'b|biblios' => \$biblios, |
60 |
'b|biblios' => \$biblios, |
58 |
'a|authorities' => \$authorities, |
61 |
'a|authorities' => \$authorities, |
|
|
62 |
'authtypes:s' => \$authtypes, |
63 |
'filter=s@' => \$filters, |
64 |
'insert' => \$insert, |
65 |
'update' => \$update, |
66 |
'all' => \$all, |
59 |
'match=s@' => \$match, |
67 |
'match=s@' => \$match, |
60 |
'i|isbn' => \$isbn_check, |
68 |
'i|isbn' => \$isbn_check, |
61 |
'x:s' => \$sourcetag, |
69 |
'x:s' => \$sourcetag, |
62 |
'y:s' => \$sourcesubfield, |
70 |
'y:s' => \$sourcesubfield, |
63 |
'idmap:s' => \$idmapfl, |
71 |
'idmap:s' => \$idmapfl, |
64 |
'cleanisbn!' => \$cleanisbn, |
72 |
'cleanisbn!' => \$cleanisbn, |
|
|
73 |
'yaml:s' => \$yamlfile, |
65 |
'dedupbarcode' => \$dedup_barcode, |
74 |
'dedupbarcode' => \$dedup_barcode, |
66 |
'framework=s' => \$framework, |
75 |
'framework=s' => \$framework, |
67 |
); |
76 |
); |
68 |
$biblios=!$authorities||$biblios; |
77 |
$biblios ||= !$authorities; |
|
|
78 |
$insert ||= !$update; |
79 |
|
80 |
if ($all) { |
81 |
$insert = 1; |
82 |
$update = 1; |
83 |
} |
69 |
|
84 |
|
70 |
if ($version || ($input_marc_file eq '')) { |
85 |
if ($version || ($input_marc_file eq '')) { |
71 |
pod2usage( -verbose => 2 ); |
86 |
pod2usage( -verbose => 2 ); |
72 |
exit; |
87 |
exit; |
73 |
} |
88 |
} |
74 |
|
89 |
|
|
|
90 |
my $dbh = C4::Context->dbh; |
91 |
my $heading_fields=get_heading_fields(); |
92 |
|
75 |
if (defined $idmapfl) { |
93 |
if (defined $idmapfl) { |
76 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
94 |
open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; |
77 |
} |
95 |
} |
Lines 81-88
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
Link Here
|
81 |
$sourcesubfield="a"; |
99 |
$sourcesubfield="a"; |
82 |
} |
100 |
} |
83 |
|
101 |
|
84 |
my $dbh = C4::Context->dbh; |
|
|
85 |
|
86 |
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & |
102 |
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & |
87 |
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user. |
103 |
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user. |
88 |
my $CataloguingLog = C4::Context->preference('CataloguingLog'); |
104 |
my $CataloguingLog = C4::Context->preference('CataloguingLog'); |
Lines 142-148
$batch->warnings_off();
Link Here
|
142 |
$batch->strict_off(); |
158 |
$batch->strict_off(); |
143 |
my $i=0; |
159 |
my $i=0; |
144 |
my $commitnum = $commit ? $commit : 50; |
160 |
my $commitnum = $commit ? $commit : 50; |
145 |
|
161 |
my $yamlhash; |
146 |
|
162 |
|
147 |
# Skip file offset |
163 |
# Skip file offset |
148 |
if ( $offset ) { |
164 |
if ( $offset ) { |
Lines 199-204
RECORD: while ( ) {
Link Here
|
199 |
next RECORD; |
215 |
next RECORD; |
200 |
} |
216 |
} |
201 |
} |
217 |
} |
|
|
218 |
SetUTF8Flag($record); |
202 |
my $isbn; |
219 |
my $isbn; |
203 |
# remove trailing - in isbn (only for biblios, of course) |
220 |
# remove trailing - in isbn (only for biblios, of course) |
204 |
if ($biblios && $cleanisbn) { |
221 |
if ($biblios && $cleanisbn) { |
Lines 212-256
RECORD: while ( ) {
Link Here
|
212 |
} |
229 |
} |
213 |
my $id; |
230 |
my $id; |
214 |
# search for duplicates (based on Local-number) |
231 |
# search for duplicates (based on Local-number) |
215 |
if ($match){ |
232 |
my $originalid; |
216 |
require C4::Search; |
233 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
217 |
my $query=build_query($match,$record); |
234 |
if ($match) { |
218 |
my $server=($authorities?'authorityserver':'biblioserver'); |
235 |
require C4::Search; |
219 |
my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
236 |
my $query = build_query( $match, $record ); |
220 |
die "unable to search the database for duplicates : $error" if (defined $error); |
237 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
221 |
#warn "$query $server : $totalhits"; |
238 |
$debug && warn $query; |
222 |
if ( @{$results} == 1 ){ |
239 |
my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); |
223 |
my $marcrecord = MARC::File::USMARC::decode($results->[0]); |
240 |
die "unable to search the database for duplicates : $error" if ( defined $error ); |
224 |
$id=GetRecordId($marcrecord,$tagid,$subfieldid); |
241 |
$debug && warn "$query $server : $totalhits"; |
225 |
} |
242 |
if ( $results && scalar(@$results) == 1 ) { |
226 |
elsif ( @{$results} > 1){ |
243 |
my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); |
227 |
$debug && warn "more than one match for $query"; |
244 |
SetUTF8Flag($marcrecord); |
228 |
} |
245 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
229 |
else { |
246 |
if ( $authorities && $marcFlavour ) { |
230 |
$debug && warn "nomatch for $query"; |
247 |
#Skip if authority in database is the same as the on in database |
231 |
} |
248 |
if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { |
|
|
249 |
if ($yamlfile) { |
250 |
$yamlhash->{$originalid}->{'authid'} = $id; |
251 |
|
252 |
# we recover all subfields of the heading authorities |
253 |
my @subfields; |
254 |
foreach my $field ( $marcrecord->field("2..") ) { |
255 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
256 |
} |
257 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
258 |
} |
259 |
next; |
260 |
} |
261 |
} |
262 |
} elsif ( $results && scalar(@$results) > 1 ) { |
263 |
$debug && warn "more than one match for $query"; |
264 |
} else { |
265 |
$debug && warn "nomatch for $query"; |
266 |
} |
232 |
} |
267 |
} |
233 |
my $originalid; |
268 |
if ($keepids && $originalid) { |
234 |
if ($keepids){ |
269 |
my $storeidfield; |
235 |
$originalid=GetRecordId($record,$tagid,$subfieldid); |
270 |
if ( length($keepids) == 3 ) { |
236 |
if ($originalid){ |
271 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
237 |
my $storeidfield; |
272 |
} else { |
238 |
if (length($keepids)==3){ |
273 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
239 |
$storeidfield=MARC::Field->new($keepids,$originalid); |
274 |
} |
240 |
} |
275 |
$record->insert_fields_ordered($storeidfield); |
241 |
else { |
276 |
$record->delete_field( $record->field($tagid) ); |
242 |
$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); |
277 |
} |
243 |
} |
278 |
foreach my $stringfilter (@$filters) { |
244 |
$record->insert_fields_ordered($storeidfield); |
279 |
if ( length($stringfilter) == 3 ) { |
245 |
$record->delete_field($record->field($tagid)); |
280 |
foreach my $field ( $record->field($stringfilter) ) { |
246 |
} |
281 |
$record->delete_field($field); |
|
|
282 |
$debug && warn "removed : ", $field->as_string; |
283 |
} |
284 |
} else { |
285 |
my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) |
286 |
if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; |
287 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
288 |
foreach my $field ( $record->field($removetag) ) { |
289 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
290 |
$debug && warn "Potentially removed : ", $field->subfield($removesubfield); |
291 |
} |
292 |
} |
293 |
} |
247 |
} |
294 |
} |
248 |
unless ($test_parameter) { |
295 |
unless ($test_parameter) { |
249 |
if ($authorities){ |
296 |
if ($authorities){ |
250 |
use C4::AuthoritiesMarc; |
297 |
use C4::AuthoritiesMarc; |
251 |
my $authtypecode=GuessAuthTypeCode($record); |
298 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
252 |
my $authid= ($id?$id:GuessAuthId($record)); |
299 |
my $authid= ($id?$id:GuessAuthId($record)); |
253 |
if ($authid && GetAuthority($authid)){ |
300 |
if ($authid && GetAuthority($authid) && $update ){ |
254 |
## Authority has an id and is in database : Replace |
301 |
## Authority has an id and is in database : Replace |
255 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
302 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
256 |
if ($@){ |
303 |
if ($@){ |
Lines 283-288
RECORD: while ( ) {
Link Here
|
283 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
330 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
284 |
} |
331 |
} |
285 |
} |
332 |
} |
|
|
333 |
if ($yamlfile) { |
334 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
335 |
my @subfields; |
336 |
foreach my $field ( $record->field("2..") ) { |
337 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
338 |
} |
339 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
340 |
} |
286 |
} |
341 |
} |
287 |
else { |
342 |
else { |
288 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
343 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
Lines 306-325
RECORD: while ( ) {
Link Here
|
306 |
} |
361 |
} |
307 |
# create biblio, unless we already have it ( either match or isbn ) |
362 |
# create biblio, unless we already have it ( either match or isbn ) |
308 |
if ($biblionumber) { |
363 |
if ($biblionumber) { |
|
|
364 |
<<<<<<< HEAD |
309 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} |
365 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} |
310 |
} |
366 |
} |
311 |
else |
367 |
else |
312 |
{ |
368 |
{ |
313 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; |
369 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; |
|
|
370 |
======= |
371 |
eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; |
372 |
if ($update) { |
373 |
eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; |
374 |
if ($@) { |
375 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
376 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
377 |
next RECORD; |
378 |
} else { |
379 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
380 |
} |
381 |
} else { |
382 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
383 |
} |
384 |
} else { |
385 |
if ($insert) { |
386 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; |
387 |
if ($@) { |
388 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
389 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
390 |
next RECORD; |
391 |
} else { |
392 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
393 |
} |
394 |
} else { |
395 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
396 |
} |
397 |
>>>>>>> Bug 5635: bulkmarcimport new parameters & features |
314 |
} |
398 |
} |
315 |
if ( $@ ) { |
|
|
316 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
317 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); |
318 |
next RECORD; |
319 |
} |
320 |
else{ |
321 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); |
322 |
} |
323 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
399 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
324 |
my $error_adding = $@; |
400 |
my $error_adding = $@; |
325 |
# Work on a clone so that if there are real errors, we can maybe |
401 |
# Work on a clone so that if there are real errors, we can maybe |
Lines 384-389
RECORD: while ( ) {
Link Here
|
384 |
if ($#{ $errors_ref } > -1) { |
460 |
if ($#{ $errors_ref } > -1) { |
385 |
report_item_errors($biblionumber, $errors_ref); |
461 |
report_item_errors($biblionumber, $errors_ref); |
386 |
} |
462 |
} |
|
|
463 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
387 |
} |
464 |
} |
388 |
$dbh->commit() if (0 == $i % $commitnum); |
465 |
$dbh->commit() if (0 == $i % $commitnum); |
389 |
} |
466 |
} |
Lines 407-412
if ($logfile){
Link Here
|
407 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
484 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
408 |
$loghandle->close; |
485 |
$loghandle->close; |
409 |
} |
486 |
} |
|
|
487 |
if ($yamlfile) { |
488 |
open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; |
489 |
print YAML Dump($yamlhash); |
490 |
} |
410 |
exit 0; |
491 |
exit 0; |
411 |
|
492 |
|
412 |
sub GetRecordId{ |
493 |
sub GetRecordId{ |
Lines 462-470
sub report_item_errors {
Link Here
|
462 |
} |
543 |
} |
463 |
sub printlog{ |
544 |
sub printlog{ |
464 |
my $logelements=shift; |
545 |
my $logelements=shift; |
465 |
print $loghandle join (";",@$logelements{qw<id op status>}),"\n"; |
546 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
|
|
547 |
} |
548 |
sub get_heading_fields{ |
549 |
my $headingfields; |
550 |
if ($authtypes){ |
551 |
$headingfields=YAML::LoadFile($authtypes); |
552 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
553 |
$debug && warn YAML::Dump($headingfields); |
554 |
} |
555 |
unless ($headingfields){ |
556 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
557 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
558 |
} |
559 |
return $headingfields; |
466 |
} |
560 |
} |
467 |
|
|
|
468 |
|
561 |
|
469 |
=head1 NAME |
562 |
=head1 NAME |
470 |
|
563 |
|
Lines 526-532
The I<NUMBER> of records to wait before performing a 'commit' operation
Link Here
|
526 |
|
619 |
|
527 |
File logs actions done for each record and their status into file |
620 |
File logs actions done for each record and their status into file |
528 |
|
621 |
|
529 |
=item B<-t> |
622 |
=item B<-t, -test> |
530 |
|
623 |
|
531 |
Test mode: parses the file, saying what he would do, but doing nothing. |
624 |
Test mode: parses the file, saying what he would do, but doing nothing. |
532 |
|
625 |
|
Lines 549-554
biblioitems, items
Link Here
|
549 |
|
642 |
|
550 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
643 |
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) |
551 |
|
644 |
|
|
|
645 |
=item B<-authtypes> |
646 |
|
647 |
file yamlfile with authoritiesTypes and distinguishable record field in order |
648 |
to store the correct authtype |
649 |
|
650 |
=item B<-yaml> |
651 |
|
652 |
yaml file format a yaml file with ids |
653 |
keepids field store ids in field (usefull for authorities, |
654 |
where 001 contains the authid for Koha, that can contain a very valuable |
655 |
info for authorities coming from LOC or BNF. useless for biblios probably |
656 |
|
657 |
=item B<-insert> |
658 |
|
659 |
if set, only insert when possible |
660 |
|
661 |
=item B<-update> |
662 |
|
663 |
if set, only updates (any biblio should have a matching record) |
664 |
|
665 |
=item B<-all> |
666 |
|
667 |
if set, do whatever is required |
668 |
|
552 |
=item B<-k, -keepids>=<FIELD> |
669 |
=item B<-k, -keepids>=<FIELD> |
553 |
|
670 |
|
554 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
671 |
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the |
555 |
- |
|
|