View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-2 / +2 lines)
Lines 447-455 Get the record and tries to guess the adequate authtypecode from its content. Link Here
447
=cut
447
=cut
448
448
449
sub GuessAuthTypeCode {
449
sub GuessAuthTypeCode {
450
    my ($record) = @_;
450
    my ($record, $heading_fields) = @_;
451
    return unless defined $record;
451
    return unless defined $record;
452
my $heading_fields = {
452
    $heading_fields //= {
453
    "MARC21"=>{
453
    "MARC21"=>{
454
        '100'=>{authtypecode=>'PERSO_NAME'},
454
        '100'=>{authtypecode=>'PERSO_NAME'},
455
        '110'=>{authtypecode=>'CORPO_NAME'},
455
        '110'=>{authtypecode=>'CORPO_NAME'},
(-)a/misc/migration_tools/bulkmarcimport.pl (-54 / +163 lines)
Lines 24-38 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
33
binmode STDOUT, ':encoding(UTF-8)';
34
use open qw( :std :utf8 );
35
binmode( STDOUT, ":utf8" );
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
36
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
37
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
38
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
36
my $cleanisbn = 1;
39
my $cleanisbn = 1;
37
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
40
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
38
41
Lines 45-51 GetOptions( Link Here
45
    'o|offset:f' => \$offset,
48
    'o|offset:f' => \$offset,
46
    'h' => \$version,
49
    'h' => \$version,
47
    'd' => \$delete,
50
    'd' => \$delete,
48
    't' => \$test_parameter,
51
    't|test' => \$test_parameter,
49
    's' => \$skip_marc8_conversion,
52
    's' => \$skip_marc8_conversion,
50
    'c:s' => \$char_encoding,
53
    'c:s' => \$char_encoding,
51
    'v:s' => \$verbose,
54
    'v:s' => \$verbose,
Lines 55-75 GetOptions( Link Here
55
    'k|keepids:s' => \$keepids,
58
    'k|keepids:s' => \$keepids,
56
    'b|biblios' => \$biblios,
59
    'b|biblios' => \$biblios,
57
    'a|authorities' => \$authorities,
60
    'a|authorities' => \$authorities,
61
    'authtypes:s' => \$authtypes,
62
    'filter=s@'     => \$filters,
63
    'insert'        => \$insert,
64
    'update'        => \$update,
65
    'all'           => \$all,
58
    'match=s@'    => \$match,
66
    'match=s@'    => \$match,
59
    'i|isbn' => \$isbn_check,
67
    'i|isbn' => \$isbn_check,
60
    'x:s' => \$sourcetag,
68
    'x:s' => \$sourcetag,
61
    'y:s' => \$sourcesubfield,
69
    'y:s' => \$sourcesubfield,
62
    'idmap:s' => \$idmapfl,
70
    'idmap:s' => \$idmapfl,
63
    'cleanisbn!'     => \$cleanisbn,
71
    'cleanisbn!'     => \$cleanisbn,
72
    'yaml:s'        => \$yamlfile,
64
    'dedupbarcode' => \$dedup_barcode,
73
    'dedupbarcode' => \$dedup_barcode,
65
);
74
);
66
$biblios=!$authorities||$biblios;
75
$biblios ||= !$authorities;
76
$insert  ||= !$update;
77
78
if ($all) {
79
    $insert = 1;
80
    $update = 1;
81
}
67
82
68
if ($version || ($input_marc_file eq '')) {
83
if ($version || ($input_marc_file eq '')) {
69
    pod2usage( -verbose => 2 );
84
    pod2usage( -verbose => 2 );
70
    exit;
85
    exit;
71
}
86
}
72
87
88
my $heading_fields=get_heading_fields();
89
73
if (defined $idmapfl) {
90
if (defined $idmapfl) {
74
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
91
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
75
}
92
}
Lines 140-146 $batch->warnings_off(); Link Here
140
$batch->strict_off();
157
$batch->strict_off();
141
my $i=0;
158
my $i=0;
142
my $commitnum = $commit ? $commit : 50;
159
my $commitnum = $commit ? $commit : 50;
143
160
my $yamlhash;
144
161
145
# Skip file offset
162
# Skip file offset
146
if ( $offset ) {
163
if ( $offset ) {
Lines 197-202 RECORD: while ( ) { Link Here
197
            next RECORD;            
214
            next RECORD;            
198
        }
215
        }
199
    }
216
    }
217
    SetUTF8Flag($record);
200
    my $isbn;
218
    my $isbn;
201
    # remove trailing - in isbn (only for biblios, of course)
219
    # remove trailing - in isbn (only for biblios, of course)
202
    if ($biblios && $cleanisbn) {
220
    if ($biblios && $cleanisbn) {
Lines 210-254 RECORD: while ( ) { Link Here
210
    }
228
    }
211
    my $id;
229
    my $id;
212
    # search for duplicates (based on Local-number)
230
    # search for duplicates (based on Local-number)
213
    if ($match){
231
    my $originalid;
214
       require C4::Search;
232
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
215
       my $query=build_query($match,$record);
233
    if ($match) {
216
       my $server=($authorities?'authorityserver':'biblioserver');
234
        require C4::Search;
217
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
235
        my $query = build_query( $match, $record );
218
       die "unable to search the database for duplicates : $error" if (defined $error);
236
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
219
       #warn "$query $server : $totalhits";
237
        $debug && warn $query;
220
       if ( @{$results} == 1 ){
238
        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
221
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
239
        die "unable to search the database for duplicates : $error" if ( defined $error );
222
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
240
        $debug && warn "$query $server : $totalhits";
223
       } 
241
        if ( $results && scalar(@$results) == 1 ) {
224
       elsif  ( @{$results} > 1){
242
            my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
225
           $debug && warn "more than one match for $query";
243
            SetUTF8Flag($marcrecord);
226
       } 
244
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
227
       else {
245
            if ( $authorities && $marcFlavour ) {
228
           $debug && warn "nomatch for $query";
246
                #Skip if authority in database is the same as the on in database
229
       }
247
                if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
248
                    if ($yamlfile) {
249
                        $yamlhash->{$originalid}->{'authid'} = $id;
250
251
                        # On récupère tous les souschamps des champs vedettes d'autorités
252
                        my @subfields;
253
                        foreach my $field ( $marcrecord->field("2..") ) {
254
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
255
                        }
256
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
257
                    }
258
                    next;
259
                }
260
            }
261
        } elsif ( $results && scalar(@$results) > 1 ) {
262
            $debug && warn "more than one match for $query";
263
        } else {
264
            $debug && warn "nomatch for $query";
265
        }
230
    }
266
    }
231
	my $originalid;
267
    if ($keepids && $originalid) {
232
    if ($keepids){
268
            my $storeidfield;
233
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
269
            if ( length($keepids) == 3 ) {
234
      if ($originalid){
270
                $storeidfield = MARC::Field->new( $keepids, $originalid );
235
		 my $storeidfield;
271
            } else {
236
		 if (length($keepids)==3){
272
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
237
		 	$storeidfield=MARC::Field->new($keepids,$originalid);
273
            }
238
		 }
274
            $record->insert_fields_ordered($storeidfield);
239
		 else  {
275
            $record->delete_field( $record->field($tagid) );
240
			$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
276
    }
241
		 }
277
    foreach my $stringfilter (@$filters) {
242
         $record->insert_fields_ordered($storeidfield);
278
        if ( length($stringfilter) == 3 ) {
243
	     $record->delete_field($record->field($tagid));
279
            foreach my $field ( $record->field($stringfilter) ) {
244
      }
280
                $record->delete_field($field);
281
                $debug && warn "removed : ", $field->as_string;
282
            }
283
        } else {
284
            my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 )
285
              if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/;
286
            if ( ( $removetag > "010" ) && $removesubfield ) {
287
                foreach my $field ( $record->field($removetag) ) {
288
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
289
                    $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
290
                }
291
            }
292
        }
245
    }
293
    }
246
    unless ($test_parameter) {
294
    unless ($test_parameter) {
247
        if ($authorities){
295
        if ($authorities){
248
            use C4::AuthoritiesMarc;
296
            use C4::AuthoritiesMarc;
249
            my $authtypecode=GuessAuthTypeCode($record);
297
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
250
            my $authid= ($id?$id:GuessAuthId($record));
298
            my $authid= ($id?$id:GuessAuthId($record));
251
            if ($authid && GetAuthority($authid)){
299
            if ($authid && GetAuthority($authid) && $update ){
252
            ## Authority has an id and is in database : Replace
300
            ## Authority has an id and is in database : Replace
253
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
301
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
254
                if ($@){
302
                if ($@){
Lines 281-286 RECORD: while ( ) { Link Here
281
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
329
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
282
				}
330
				}
283
 	        }
331
 	        }
332
            if ($yamlfile) {
333
            $yamlhash->{$originalid}->{'authid'} = $authid;
334
            my @subfields;
335
            foreach my $field ( $record->field("2..") ) {
336
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
337
            }
338
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
339
            }
284
        }
340
        }
285
        else {
341
        else {
286
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
342
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 304-323 RECORD: while ( ) { Link Here
304
			}
360
			}
305
					# create biblio, unless we already have it ( either match or isbn )
361
					# create biblio, unless we already have it ( either match or isbn )
306
            if ($biblionumber) {
362
            if ($biblionumber) {
307
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
363
                eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
308
			}
364
                if ($update) {
309
			else 
365
                    eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
310
			{
366
                    if ($@) {
311
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) };
367
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
368
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
369
                        next RECORD;
370
                    } else {
371
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
372
                    }
373
                } else {
374
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
375
                }
376
            } else {
377
                if ($insert) {
378
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
379
                    if ($@) {
380
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
381
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
382
                        next RECORD;
383
                    } else {
384
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
385
                    }
386
                } else {
387
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
388
                }
312
            }
389
            }
313
            if ( $@ ) {
314
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
315
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
316
                next RECORD;
317
            } 
318
 			else{
319
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
320
			}
321
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
390
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
322
            my $error_adding = $@;
391
            my $error_adding = $@;
323
            # Work on a clone so that if there are real errors, we can maybe
392
            # Work on a clone so that if there are real errors, we can maybe
Lines 382-387 RECORD: while ( ) { Link Here
382
            if ($#{ $errors_ref } > -1) {
451
            if ($#{ $errors_ref } > -1) {
383
                report_item_errors($biblionumber, $errors_ref);
452
                report_item_errors($biblionumber, $errors_ref);
384
            }
453
            }
454
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
385
        }
455
        }
386
        $dbh->commit() if (0 == $i % $commitnum);
456
        $dbh->commit() if (0 == $i % $commitnum);
387
    }
457
    }
Lines 405-410 if ($logfile){ Link Here
405
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
475
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
406
  $loghandle->close;
476
  $loghandle->close;
407
}
477
}
478
if ($yamlfile) {
479
    open YAML, "> $yamlfile" or die "cannot open $yamlfile \n";
480
    print YAML Dump($yamlhash);
481
}
408
exit 0;
482
exit 0;
409
483
410
sub GetRecordId{
484
sub GetRecordId{
Lines 460-468 sub report_item_errors { Link Here
460
}
534
}
461
sub printlog{
535
sub printlog{
462
	my $logelements=shift;
536
	my $logelements=shift;
463
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
537
	print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
538
}
539
sub get_heading_fields{
540
    my $headingfields;
541
    if ($authtypes){
542
        $headingfields=YAML::LoadFile($authtypes);
543
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
544
        $debug && warn YAML::Dump($headingfields);
545
    }
546
    unless ($headingfields){
547
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
548
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
549
    }
550
    return $headingfields;
464
}
551
}
465
466
552
467
=head1 NAME
553
=head1 NAME
468
554
Lines 524-530 The I<NUMBER> of records to wait before performing a 'commit' operation Link Here
524
610
525
File logs actions done for each record and their status into file
611
File logs actions done for each record and their status into file
526
612
527
=item B<-t>
613
=item B<-t, -test>
528
614
529
Test mode: parses the file, saying what he would do, but doing nothing.
615
Test mode: parses the file, saying what he would do, but doing nothing.
530
616
Lines 547-552 biblioitems, items Link Here
547
633
548
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
634
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
549
635
636
=item B<-authtypes>
637
638
file yamlfile with authoritiesTypes and distinguishable record field in order
639
to store the correct authtype
640
641
=item B<-yaml>
642
643
yaml file  format a yaml file with ids
644
keepids field store ids in field (usefull for authorities,
645
where 001 contains the authid for Koha, that can contain a very valuable
646
info for authorities coming from LOC or BNF. useless for biblios probably
647
648
=item B<-insert>
649
650
if set, only insert when possible
651
652
=item B<-update>
653
654
if set, only updates (any biblio should have a matching record)
655
656
=item B<-all>
657
658
if set, do whatever is required
659
550
=item B<-k, -keepids>=<FIELD>
660
=item B<-k, -keepids>=<FIELD>
551
661
552
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
662
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
553
- 

Return to bug 5635