View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-2 / +2 lines)
Lines 456-464 Get the record and tries to guess the adequate authtypecode from its content. Link Here
456
=cut
456
=cut
457
457
458
sub GuessAuthTypeCode {
458
sub GuessAuthTypeCode {
459
    my ($record) = @_;
459
    my ($record, $heading_fields) = @_;
460
    return unless defined $record;
460
    return unless defined $record;
461
my $heading_fields = {
461
    $heading_fields //= {
462
    "MARC21"=>{
462
    "MARC21"=>{
463
        '100'=>{authtypecode=>'PERSO_NAME'},
463
        '100'=>{authtypecode=>'PERSO_NAME'},
464
        '110'=>{authtypecode=>'CORPO_NAME'},
464
        '110'=>{authtypecode=>'CORPO_NAME'},
(-)a/misc/migration_tools/bulkmarcimport.pl (-54 / +163 lines)
Lines 24-38 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
33
binmode STDOUT, ':encoding(UTF-8)';
34
use open qw( :std :utf8 );
35
binmode( STDOUT, ":utf8" );
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
36
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
37
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
38
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
36
my $cleanisbn = 1;
39
my $cleanisbn = 1;
37
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
40
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
38
my $framework = '';
41
my $framework = '';
Lines 46-52 GetOptions( Link Here
46
    'o|offset:f' => \$offset,
49
    'o|offset:f' => \$offset,
47
    'h' => \$version,
50
    'h' => \$version,
48
    'd' => \$delete,
51
    'd' => \$delete,
49
    't' => \$test_parameter,
52
    't|test' => \$test_parameter,
50
    's' => \$skip_marc8_conversion,
53
    's' => \$skip_marc8_conversion,
51
    'c:s' => \$char_encoding,
54
    'c:s' => \$char_encoding,
52
    'v:s' => \$verbose,
55
    'v:s' => \$verbose,
Lines 56-77 GetOptions( Link Here
56
    'k|keepids:s' => \$keepids,
59
    'k|keepids:s' => \$keepids,
57
    'b|biblios' => \$biblios,
60
    'b|biblios' => \$biblios,
58
    'a|authorities' => \$authorities,
61
    'a|authorities' => \$authorities,
62
    'authtypes:s' => \$authtypes,
63
    'filter=s@'     => \$filters,
64
    'insert'        => \$insert,
65
    'update'        => \$update,
66
    'all'           => \$all,
59
    'match=s@'    => \$match,
67
    'match=s@'    => \$match,
60
    'i|isbn' => \$isbn_check,
68
    'i|isbn' => \$isbn_check,
61
    'x:s' => \$sourcetag,
69
    'x:s' => \$sourcetag,
62
    'y:s' => \$sourcesubfield,
70
    'y:s' => \$sourcesubfield,
63
    'idmap:s' => \$idmapfl,
71
    'idmap:s' => \$idmapfl,
64
    'cleanisbn!'     => \$cleanisbn,
72
    'cleanisbn!'     => \$cleanisbn,
73
    'yaml:s'        => \$yamlfile,
65
    'dedupbarcode' => \$dedup_barcode,
74
    'dedupbarcode' => \$dedup_barcode,
66
    'framework=s' => \$framework,
75
    'framework=s' => \$framework,
67
);
76
);
68
$biblios=!$authorities||$biblios;
77
$biblios ||= !$authorities;
78
$insert  ||= !$update;
79
80
if ($all) {
81
    $insert = 1;
82
    $update = 1;
83
}
69
84
70
if ($version || ($input_marc_file eq '')) {
85
if ($version || ($input_marc_file eq '')) {
71
    pod2usage( -verbose => 2 );
86
    pod2usage( -verbose => 2 );
72
    exit;
87
    exit;
73
}
88
}
74
89
90
my $heading_fields=get_heading_fields();
91
75
if (defined $idmapfl) {
92
if (defined $idmapfl) {
76
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
93
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
77
}
94
}
Lines 142-148 $batch->warnings_off(); Link Here
142
$batch->strict_off();
159
$batch->strict_off();
143
my $i=0;
160
my $i=0;
144
my $commitnum = $commit ? $commit : 50;
161
my $commitnum = $commit ? $commit : 50;
145
162
my $yamlhash;
146
163
147
# Skip file offset
164
# Skip file offset
148
if ( $offset ) {
165
if ( $offset ) {
Lines 199-204 RECORD: while ( ) { Link Here
199
            next RECORD;            
216
            next RECORD;            
200
        }
217
        }
201
    }
218
    }
219
    SetUTF8Flag($record);
202
    my $isbn;
220
    my $isbn;
203
    # remove trailing - in isbn (only for biblios, of course)
221
    # remove trailing - in isbn (only for biblios, of course)
204
    if ($biblios && $cleanisbn) {
222
    if ($biblios && $cleanisbn) {
Lines 212-256 RECORD: while ( ) { Link Here
212
    }
230
    }
213
    my $id;
231
    my $id;
214
    # search for duplicates (based on Local-number)
232
    # search for duplicates (based on Local-number)
215
    if ($match){
233
    my $originalid;
216
       require C4::Search;
234
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
217
       my $query=build_query($match,$record);
235
    if ($match) {
218
       my $server=($authorities?'authorityserver':'biblioserver');
236
        require C4::Search;
219
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
237
        my $query = build_query( $match, $record );
220
       die "unable to search the database for duplicates : $error" if (defined $error);
238
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
221
       #warn "$query $server : $totalhits";
239
        $debug && warn $query;
222
       if ( @{$results} == 1 ){
240
        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
223
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
241
        die "unable to search the database for duplicates : $error" if ( defined $error );
224
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
242
        $debug && warn "$query $server : $totalhits";
225
       } 
243
        if ( $results && scalar(@$results) == 1 ) {
226
       elsif  ( @{$results} > 1){
244
            my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
227
           $debug && warn "more than one match for $query";
245
            SetUTF8Flag($marcrecord);
228
       } 
246
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
229
       else {
247
            if ( $authorities && $marcFlavour ) {
230
           $debug && warn "nomatch for $query";
248
                #Skip if authority in database is the same as the on in database
231
       }
249
                if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
250
                    if ($yamlfile) {
251
                        $yamlhash->{$originalid}->{'authid'} = $id;
252
253
                        # we recover all subfields of the heading authorities
254
                        my @subfields;
255
                        foreach my $field ( $marcrecord->field("2..") ) {
256
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
257
                        }
258
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
259
                    }
260
                    next;
261
                }
262
            }
263
        } elsif ( $results && scalar(@$results) > 1 ) {
264
            $debug && warn "more than one match for $query";
265
        } else {
266
            $debug && warn "nomatch for $query";
267
        }
232
    }
268
    }
233
	my $originalid;
269
    if ($keepids && $originalid) {
234
    if ($keepids){
270
            my $storeidfield;
235
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
271
            if ( length($keepids) == 3 ) {
236
      if ($originalid){
272
                $storeidfield = MARC::Field->new( $keepids, $originalid );
237
		 my $storeidfield;
273
            } else {
238
		 if (length($keepids)==3){
274
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
239
		 	$storeidfield=MARC::Field->new($keepids,$originalid);
275
            }
240
		 }
276
            $record->insert_fields_ordered($storeidfield);
241
		 else  {
277
            $record->delete_field( $record->field($tagid) );
242
			$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
278
    }
243
		 }
279
    foreach my $stringfilter (@$filters) {
244
         $record->insert_fields_ordered($storeidfield);
280
        if ( length($stringfilter) == 3 ) {
245
	     $record->delete_field($record->field($tagid));
281
            foreach my $field ( $record->field($stringfilter) ) {
246
      }
282
                $record->delete_field($field);
283
                $debug && warn "removed : ", $field->as_string;
284
            }
285
        } else {
286
            my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 )
287
              if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/;
288
            if ( ( $removetag > "010" ) && $removesubfield ) {
289
                foreach my $field ( $record->field($removetag) ) {
290
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
291
                    $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
292
                }
293
            }
294
        }
247
    }
295
    }
248
    unless ($test_parameter) {
296
    unless ($test_parameter) {
249
        if ($authorities){
297
        if ($authorities){
250
            use C4::AuthoritiesMarc;
298
            use C4::AuthoritiesMarc;
251
            my $authtypecode=GuessAuthTypeCode($record);
299
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
252
            my $authid= ($id?$id:GuessAuthId($record));
300
            my $authid= ($id?$id:GuessAuthId($record));
253
            if ($authid && GetAuthority($authid)){
301
            if ($authid && GetAuthority($authid) && $update ){
254
            ## Authority has an id and is in database : Replace
302
            ## Authority has an id and is in database : Replace
255
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
303
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
256
                if ($@){
304
                if ($@){
Lines 283-288 RECORD: while ( ) { Link Here
283
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
331
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
284
				}
332
				}
285
 	        }
333
 	        }
334
            if ($yamlfile) {
335
            $yamlhash->{$originalid}->{'authid'} = $authid;
336
            my @subfields;
337
            foreach my $field ( $record->field("2..") ) {
338
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
339
            }
340
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
341
            }
286
        }
342
        }
287
        else {
343
        else {
288
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
344
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 306-325 RECORD: while ( ) { Link Here
306
			}
362
			}
307
					# create biblio, unless we already have it ( either match or isbn )
363
					# create biblio, unless we already have it ( either match or isbn )
308
            if ($biblionumber) {
364
            if ($biblionumber) {
309
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
365
                eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
310
			}
366
                if ($update) {
311
			else 
367
                    eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
312
			{
368
                    if ($@) {
313
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
369
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
370
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
371
                        next RECORD;
372
                    } else {
373
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
374
                    }
375
                } else {
376
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
377
                }
378
            } else {
379
                if ($insert) {
380
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
381
                    if ($@) {
382
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
383
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
384
                        next RECORD;
385
                    } else {
386
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
387
                    }
388
                } else {
389
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
390
                }
314
            }
391
            }
315
            if ( $@ ) {
316
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
317
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
318
                next RECORD;
319
            } 
320
 			else{
321
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
322
			}
323
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
392
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
324
            my $error_adding = $@;
393
            my $error_adding = $@;
325
            # Work on a clone so that if there are real errors, we can maybe
394
            # Work on a clone so that if there are real errors, we can maybe
Lines 384-389 RECORD: while ( ) { Link Here
384
            if ($#{ $errors_ref } > -1) {
453
            if ($#{ $errors_ref } > -1) {
385
                report_item_errors($biblionumber, $errors_ref);
454
                report_item_errors($biblionumber, $errors_ref);
386
            }
455
            }
456
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
387
        }
457
        }
388
        $dbh->commit() if (0 == $i % $commitnum);
458
        $dbh->commit() if (0 == $i % $commitnum);
389
    }
459
    }
Lines 407-412 if ($logfile){ Link Here
407
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
477
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
408
  $loghandle->close;
478
  $loghandle->close;
409
}
479
}
480
if ($yamlfile) {
481
    open YAML, "> $yamlfile" or die "cannot open $yamlfile \n";
482
    print YAML Dump($yamlhash);
483
}
410
exit 0;
484
exit 0;
411
485
412
sub GetRecordId{
486
sub GetRecordId{
Lines 462-470 sub report_item_errors { Link Here
462
}
536
}
463
sub printlog{
537
sub printlog{
464
	my $logelements=shift;
538
	my $logelements=shift;
465
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
539
	print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
540
}
541
sub get_heading_fields{
542
    my $headingfields;
543
    if ($authtypes){
544
        $headingfields=YAML::LoadFile($authtypes);
545
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
546
        $debug && warn YAML::Dump($headingfields);
547
    }
548
    unless ($headingfields){
549
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
550
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
551
    }
552
    return $headingfields;
466
}
553
}
467
468
554
469
=head1 NAME
555
=head1 NAME
470
556
Lines 526-532 The I<NUMBER> of records to wait before performing a 'commit' operation Link Here
526
612
527
File logs actions done for each record and their status into file
613
File logs actions done for each record and their status into file
528
614
529
=item B<-t>
615
=item B<-t, -test>
530
616
531
Test mode: parses the file, saying what he would do, but doing nothing.
617
Test mode: parses the file, saying what he would do, but doing nothing.
532
618
Lines 549-554 biblioitems, items Link Here
549
635
550
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
636
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
551
637
638
=item B<-authtypes>
639
640
file yamlfile with authoritiesTypes and distinguishable record field in order
641
to store the correct authtype
642
643
=item B<-yaml>
644
645
yaml file  format a yaml file with ids
646
keepids field store ids in field (usefull for authorities,
647
where 001 contains the authid for Koha, that can contain a very valuable
648
info for authorities coming from LOC or BNF. useless for biblios probably
649
650
=item B<-insert>
651
652
if set, only insert when possible
653
654
=item B<-update>
655
656
if set, only updates (any biblio should have a matching record)
657
658
=item B<-all>
659
660
if set, do whatever is required
661
552
=item B<-k, -keepids>=<FIELD>
662
=item B<-k, -keepids>=<FIELD>
553
663
554
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
664
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
555
- 

Return to bug 5635