View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-2 / +2 lines)
Lines 412-420 Get the record and tries to guess the adequate authtypecode from its content. Link Here
412
=cut
412
=cut
413
413
414
sub GuessAuthTypeCode {
414
sub GuessAuthTypeCode {
415
    my ($record) = @_;
415
    my ($record, $heading_fields) = @_;
416
    return unless defined $record;
416
    return unless defined $record;
417
my $heading_fields = {
417
    $heading_fields //= {
418
    "MARC21"=>{
418
    "MARC21"=>{
419
        '100'=>{authtypecode=>'PERSO_NAME'},
419
        '100'=>{authtypecode=>'PERSO_NAME'},
420
        '110'=>{authtypecode=>'CORPO_NAME'},
420
        '110'=>{authtypecode=>'CORPO_NAME'},
(-)a/misc/migration_tools/bulkmarcimport.pl (-53 / +164 lines)
Lines 24-39 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
33
binmode STDOUT, ':encoding(UTF-8)';
34
use open qw( :std :utf8 );
35
binmode( STDOUT, ":utf8" );
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
36
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
37
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
36
my ($sourcetag,$sourcesubfield,$idmapfl);
38
my ($sourcetag,$sourcesubfield,$idmapfl);
39
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
37
my $cleanisbn = 1;
40
my $cleanisbn = 1;
38
41
39
$|=1;
42
$|=1;
Lines 45-51 GetOptions( Link Here
45
    'o|offset:f' => \$offset,
48
    'o|offset:f' => \$offset,
46
    'h' => \$version,
49
    'h' => \$version,
47
    'd' => \$delete,
50
    'd' => \$delete,
48
    't' => \$test_parameter,
51
    't|test' => \$test_parameter,
49
    's' => \$skip_marc8_conversion,
52
    's' => \$skip_marc8_conversion,
50
    'c:s' => \$char_encoding,
53
    'c:s' => \$char_encoding,
51
    'v:s' => \$verbose,
54
    'v:s' => \$verbose,
Lines 55-74 GetOptions( Link Here
55
    'k|keepids:s' => \$keepids,
58
    'k|keepids:s' => \$keepids,
56
    'b|biblios' => \$biblios,
59
    'b|biblios' => \$biblios,
57
    'a|authorities' => \$authorities,
60
    'a|authorities' => \$authorities,
61
    'authtypes:s' => \$authtypes,
62
    'filter=s@'     => \$filters,
63
    'insert'        => \$insert,
64
    'update'        => \$update,
65
    'all'           => \$all,
58
    'match=s@'    => \$match,
66
    'match=s@'    => \$match,
59
    'i|isbn' => \$isbn_check,
67
    'i|isbn' => \$isbn_check,
60
    'x:s' => \$sourcetag,
68
    'x:s' => \$sourcetag,
61
    'y:s' => \$sourcesubfield,
69
    'y:s' => \$sourcesubfield,
62
    'idmap:s' => \$idmapfl,
70
    'idmap:s' => \$idmapfl,
63
    'cleanisbn!'     => \$cleanisbn,
71
    'cleanisbn!'     => \$cleanisbn,
72
    'yaml:s'        => \$yamlfile,
64
);
73
);
65
$biblios=!$authorities||$biblios;
74
$biblios ||= !$authorities;
75
$insert  ||= !$update;
76
77
if ($all) {
78
    $insert = 1;
79
    $update = 1;
80
}
66
81
67
if ($version || ($input_marc_file eq '')) {
82
if ($version || ($input_marc_file eq '')) {
68
    pod2usage( -verbose => 2 );
83
    pod2usage( -verbose => 2 );
69
    exit;
84
    exit;
70
}
85
}
71
86
87
my $heading_fields=get_heading_fields();
88
72
if (defined $idmapfl) {
89
if (defined $idmapfl) {
73
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
90
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
74
}
91
}
Lines 139-145 $batch->warnings_off(); Link Here
139
$batch->strict_off();
156
$batch->strict_off();
140
my $i=0;
157
my $i=0;
141
my $commitnum = $commit ? $commit : 50;
158
my $commitnum = $commit ? $commit : 50;
142
159
my $yamlhash;
143
160
144
# Skip file offset
161
# Skip file offset
145
if ( $offset ) {
162
if ( $offset ) {
Lines 196-201 RECORD: while ( ) { Link Here
196
            next RECORD;            
213
            next RECORD;            
197
        }
214
        }
198
    }
215
    }
216
    SetUTF8Flag($record);
199
    my $isbn;
217
    my $isbn;
200
    # remove trailing - in isbn (only for biblios, of course)
218
    # remove trailing - in isbn (only for biblios, of course)
201
    if ($biblios && $cleanisbn) {
219
    if ($biblios && $cleanisbn) {
Lines 209-253 RECORD: while ( ) { Link Here
209
    }
227
    }
210
    my $id;
228
    my $id;
211
    # search for duplicates (based on Local-number)
229
    # search for duplicates (based on Local-number)
212
    if ($match){
230
    my $originalid;
213
       require C4::Search;
231
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
214
       my $query=build_query($match,$record);
232
    if ($match) {
215
       my $server=($authorities?'authorityserver':'biblioserver');
233
        require C4::Search;
216
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
234
        my $query = build_query( $match, $record );
217
       die "unable to search the database for duplicates : $error" if (defined $error);
235
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
218
       #warn "$query $server : $totalhits";
236
        $debug && warn $query;
219
       if ( @{$results} == 1 ){
237
        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
220
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
238
        die "unable to search the database for duplicates : $error" if ( defined $error );
221
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
239
        $debug && warn "$query $server : $totalhits";
222
       } 
240
        if ( $results && scalar(@$results) == 1 ) {
223
       elsif  ( @{$results} > 1){
241
            my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
224
           $debug && warn "more than one match for $query";
242
            SetUTF8Flag($marcrecord);
225
       } 
243
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
226
       else {
244
            if ( $authorities && $marcFlavour ) {
227
           $debug && warn "nomatch for $query";
245
                #Skip if authority in database is the same as the on in database
228
       }
246
                if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
247
                    if ($yamlfile) {
248
                        $yamlhash->{$originalid}->{'authid'} = $id;
249
250
                        # On récupère tous les souschamps des champs vedettes d'autorités
251
                        my @subfields;
252
                        foreach my $field ( $marcrecord->field("2..") ) {
253
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
254
                        }
255
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
256
                    }
257
                    next;
258
                }
259
            }
260
        } elsif ( $results && scalar(@$results) > 1 ) {
261
            $debug && warn "more than one match for $query";
262
        } else {
263
            $debug && warn "nomatch for $query";
264
        }
229
    }
265
    }
230
	my $originalid;
266
    if ($keepids && $originalid) {
231
    if ($keepids){
267
            my $storeidfield;
232
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
268
            if ( length($keepids) == 3 ) {
233
      if ($originalid){
269
                $storeidfield = MARC::Field->new( $keepids, $originalid );
234
		 my $storeidfield;
270
            } else {
235
		 if (length($keepids)==3){
271
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
236
		 	$storeidfield=MARC::Field->new($keepids,$originalid);
272
            }
237
		 }
273
            $record->insert_fields_ordered($storeidfield);
238
		 else  {
274
            $record->delete_field( $record->field($tagid) );
239
			$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
275
    }
240
		 }
276
    foreach my $stringfilter (@$filters) {
241
         $record->insert_fields_ordered($storeidfield);
277
        if ( length($stringfilter) == 3 ) {
242
	     $record->delete_field($record->field($tagid));
278
            foreach my $field ( $record->field($stringfilter) ) {
243
      }
279
                $record->delete_field($field);
280
                $debug && warn "removed : ", $field->as_string;
281
            }
282
        } else {
283
            my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 )
284
              if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/;
285
            if ( ( $removetag > "010" ) && $removesubfield ) {
286
                foreach my $field ( $record->field($removetag) ) {
287
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
288
                    $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
289
                }
290
            }
291
        }
244
    }
292
    }
293
245
    unless ($test_parameter) {
294
    unless ($test_parameter) {
246
        if ($authorities){
295
        if ($authorities){
247
            use C4::AuthoritiesMarc;
296
            use C4::AuthoritiesMarc;
248
            my $authtypecode=GuessAuthTypeCode($record);
297
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
249
            my $authid= ($id?$id:GuessAuthId($record));
298
            my $authid= ($id?$id:GuessAuthId($record));
250
            if ($authid && GetAuthority($authid)){
299
            if ($authid && GetAuthority($authid) && $update ){
251
            ## Authority has an id and is in database : Replace
300
            ## Authority has an id and is in database : Replace
252
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
301
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
253
                if ($@){
302
                if ($@){
Lines 280-285 RECORD: while ( ) { Link Here
280
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
329
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
281
				}
330
				}
282
 	        }
331
 	        }
332
            if ($yamlfile) {
333
            $yamlhash->{$originalid}->{'authid'} = $authid;
334
            my @subfields;
335
            foreach my $field ( $record->field("2..") ) {
336
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
337
            }
338
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
339
            }
283
        }
340
        }
284
        else {
341
        else {
285
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
342
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 303-322 RECORD: while ( ) { Link Here
303
			}
360
			}
304
					# create biblio, unless we already have it ( either match or isbn )
361
					# create biblio, unless we already have it ( either match or isbn )
305
            if ($biblionumber) {
362
            if ($biblionumber) {
306
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
363
                eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
307
			}
364
                if ($update) {
308
			else 
365
                    eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
309
			{
366
                    if ($@) {
310
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) };
367
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
368
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
369
                        next RECORD;
370
                    } else {
371
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
372
                    }
373
                } else {
374
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
375
                }
376
            } else {
377
                if ($insert) {
378
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
379
                    if ($@) {
380
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
381
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
382
                        next RECORD;
383
                    } else {
384
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
385
                    }
386
                } else {
387
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
388
                }
311
            }
389
            }
312
            if ( $@ ) {
313
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
314
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
315
                next RECORD;
316
            } 
317
 			else{
318
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
319
			}
320
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
390
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
321
            if ( $@ ) {
391
            if ( $@ ) {
322
                warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
392
                warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
Lines 335-340 RECORD: while ( ) { Link Here
335
            if ($#{ $errors_ref } > -1) { 
405
            if ($#{ $errors_ref } > -1) { 
336
                report_item_errors($biblionumber, $errors_ref);
406
                report_item_errors($biblionumber, $errors_ref);
337
            }
407
            }
408
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
338
        }
409
        }
339
        $dbh->commit() if (0 == $i % $commitnum);
410
        $dbh->commit() if (0 == $i % $commitnum);
340
    }
411
    }
Lines 358-363 if ($logfile){ Link Here
358
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
429
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
359
  $loghandle->close;
430
  $loghandle->close;
360
}
431
}
432
if ($yamlfile) {
433
    open YAML, "> $yamlfile" or die "cannot open $yamlfile \n";
434
    print YAML Dump($yamlhash);
435
}
361
exit 0;
436
exit 0;
362
437
363
sub GetRecordId{
438
sub GetRecordId{
Lines 412-420 sub report_item_errors { Link Here
412
}
487
}
413
sub printlog{
488
sub printlog{
414
	my $logelements=shift;
489
	my $logelements=shift;
415
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
490
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
416
}
491
}
492
sub get_heading_fields{
493
    my $headingfields;
494
    if ($authtypes){
495
        $headingfields=YAML::LoadFile($authtypes);
417
496
497
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
498
        $debug && warn YAML::Dump($headingfields);
499
    }
500
    unless ($headingfields){
501
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
502
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
503
    }
504
    return $headingfields;
505
}
418
506
419
=head1 NAME
507
=head1 NAME
420
508
Lines 476-482 The I<NUMBER> of records to wait before performing a 'commit' operation Link Here
476
564
477
File logs actions done for each record and their status into file
565
File logs actions done for each record and their status into file
478
566
479
=item B<-t>
567
=item B<-t, -test>
480
568
481
Test mode: parses the file, saying what he would do, but doing nothing.
569
Test mode: parses the file, saying what he would do, but doing nothing.
482
570
Lines 499-504 biblioitems, items Link Here
499
587
500
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
588
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
501
589
590
=item B<-authtypes>
591
592
file yamlfile with authoritiesTypes and distinguishable record field in order
593
to store the correct authtype
594
595
=item B<-yaml>
596
597
yaml file  format a yaml file with ids
598
keepids field store ids in field (usefull for authorities,
599
where 001 contains the authid for Koha, that can contain a very valuable
600
info for authorities coming from LOC or BNF. useless for biblios probably
601
602
=item B<-insert>
603
604
if set, only insert when possible
605
606
=item B<-update>
607
608
if set, only updates (any biblio should have a matching record)
609
610
=item B<-all>
611
612
if set, do whatever is required
613
502
=item B<-k, -keepids>=<FIELD>
614
=item B<-k, -keepids>=<FIELD>
503
615
504
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
616
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
505
- 

Return to bug 5635