View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-2 / +2 lines)
Lines 456-464 Get the record and tries to guess the adequate authtypecode from its content. Link Here
456
=cut
456
=cut
457
457
458
sub GuessAuthTypeCode {
458
sub GuessAuthTypeCode {
459
    my ($record) = @_;
459
    my ($record, $heading_fields) = @_;
460
    return unless defined $record;
460
    return unless defined $record;
461
my $heading_fields = {
461
    $heading_fields //= {
462
    "MARC21"=>{
462
    "MARC21"=>{
463
        '100'=>{authtypecode=>'PERSO_NAME'},
463
        '100'=>{authtypecode=>'PERSO_NAME'},
464
        '110'=>{authtypecode=>'CORPO_NAME'},
464
        '110'=>{authtypecode=>'CORPO_NAME'},
(-)a/misc/migration_tools/bulkmarcimport.pl (-56 / +164 lines)
Lines 24-38 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
33
binmode STDOUT, ':encoding(UTF-8)';
34
use open qw( :std :utf8 );
35
binmode( STDOUT, ":utf8" );
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
36
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
37
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
38
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
36
my $cleanisbn = 1;
39
my $cleanisbn = 1;
37
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
40
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
38
my $framework = '';
41
my $framework = '';
Lines 46-52 GetOptions( Link Here
46
    'o|offset:f' => \$offset,
49
    'o|offset:f' => \$offset,
47
    'h' => \$version,
50
    'h' => \$version,
48
    'd' => \$delete,
51
    'd' => \$delete,
49
    't' => \$test_parameter,
52
    't|test' => \$test_parameter,
50
    's' => \$skip_marc8_conversion,
53
    's' => \$skip_marc8_conversion,
51
    'c:s' => \$char_encoding,
54
    'c:s' => \$char_encoding,
52
    'v:s' => \$verbose,
55
    'v:s' => \$verbose,
Lines 56-77 GetOptions( Link Here
56
    'k|keepids:s' => \$keepids,
59
    'k|keepids:s' => \$keepids,
57
    'b|biblios' => \$biblios,
60
    'b|biblios' => \$biblios,
58
    'a|authorities' => \$authorities,
61
    'a|authorities' => \$authorities,
62
    'authtypes:s' => \$authtypes,
63
    'filter=s@'     => \$filters,
64
    'insert'        => \$insert,
65
    'update'        => \$update,
66
    'all'           => \$all,
59
    'match=s@'    => \$match,
67
    'match=s@'    => \$match,
60
    'i|isbn' => \$isbn_check,
68
    'i|isbn' => \$isbn_check,
61
    'x:s' => \$sourcetag,
69
    'x:s' => \$sourcetag,
62
    'y:s' => \$sourcesubfield,
70
    'y:s' => \$sourcesubfield,
63
    'idmap:s' => \$idmapfl,
71
    'idmap:s' => \$idmapfl,
64
    'cleanisbn!'     => \$cleanisbn,
72
    'cleanisbn!'     => \$cleanisbn,
73
    'yaml:s'        => \$yamlfile,
65
    'dedupbarcode' => \$dedup_barcode,
74
    'dedupbarcode' => \$dedup_barcode,
66
    'framework=s' => \$framework,
75
    'framework=s' => \$framework,
67
);
76
);
68
$biblios=!$authorities||$biblios;
77
$biblios ||= !$authorities;
78
$insert  ||= !$update;
79
80
if ($all) {
81
    $insert = 1;
82
    $update = 1;
83
}
69
84
70
if ($version || ($input_marc_file eq '')) {
85
if ($version || ($input_marc_file eq '')) {
71
    pod2usage( -verbose => 2 );
86
    pod2usage( -verbose => 2 );
72
    exit;
87
    exit;
73
}
88
}
74
89
90
my $dbh = C4::Context->dbh;
91
my $heading_fields=get_heading_fields();
92
75
if (defined $idmapfl) {
93
if (defined $idmapfl) {
76
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
94
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
77
}
95
}
Lines 81-88 if ((not defined $sourcesubfield) && (not defined $sourcetag)){ Link Here
81
  $sourcesubfield="a";
99
  $sourcesubfield="a";
82
}
100
}
83
101
84
my $dbh = C4::Context->dbh;
85
86
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & 
102
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & 
87
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
103
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
88
my $CataloguingLog = C4::Context->preference('CataloguingLog');
104
my $CataloguingLog = C4::Context->preference('CataloguingLog');
Lines 142-148 $batch->warnings_off(); Link Here
142
$batch->strict_off();
158
$batch->strict_off();
143
my $i=0;
159
my $i=0;
144
my $commitnum = $commit ? $commit : 50;
160
my $commitnum = $commit ? $commit : 50;
145
161
my $yamlhash;
146
162
147
# Skip file offset
163
# Skip file offset
148
if ( $offset ) {
164
if ( $offset ) {
Lines 199-204 RECORD: while ( ) { Link Here
199
            next RECORD;            
215
            next RECORD;            
200
        }
216
        }
201
    }
217
    }
218
    SetUTF8Flag($record);
202
    my $isbn;
219
    my $isbn;
203
    # remove trailing - in isbn (only for biblios, of course)
220
    # remove trailing - in isbn (only for biblios, of course)
204
    if ($biblios && $cleanisbn) {
221
    if ($biblios && $cleanisbn) {
Lines 212-256 RECORD: while ( ) { Link Here
212
    }
229
    }
213
    my $id;
230
    my $id;
214
    # search for duplicates (based on Local-number)
231
    # search for duplicates (based on Local-number)
215
    if ($match){
232
    my $originalid;
216
       require C4::Search;
233
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
217
       my $query=build_query($match,$record);
234
    if ($match) {
218
       my $server=($authorities?'authorityserver':'biblioserver');
235
        require C4::Search;
219
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
236
        my $query = build_query( $match, $record );
220
       die "unable to search the database for duplicates : $error" if (defined $error);
237
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
221
       #warn "$query $server : $totalhits";
238
        $debug && warn $query;
222
       if ( @{$results} == 1 ){
239
        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
223
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
240
        die "unable to search the database for duplicates : $error" if ( defined $error );
224
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
241
        $debug && warn "$query $server : $totalhits";
225
       } 
242
        if ( $results && scalar(@$results) == 1 ) {
226
       elsif  ( @{$results} > 1){
243
            my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
227
           $debug && warn "more than one match for $query";
244
            SetUTF8Flag($marcrecord);
228
       } 
245
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
229
       else {
246
            if ( $authorities && $marcFlavour ) {
230
           $debug && warn "nomatch for $query";
247
                #Skip if authority in database is the same as the on in database
231
       }
248
                if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
249
                    if ($yamlfile) {
250
                        $yamlhash->{$originalid}->{'authid'} = $id;
251
252
                        # we recover all subfields of the heading authorities
253
                        my @subfields;
254
                        foreach my $field ( $marcrecord->field("2..") ) {
255
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
256
                        }
257
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
258
                    }
259
                    next;
260
                }
261
            }
262
        } elsif ( $results && scalar(@$results) > 1 ) {
263
            $debug && warn "more than one match for $query";
264
        } else {
265
            $debug && warn "nomatch for $query";
266
        }
267
    }
268
    if ($keepids && $originalid) {
269
            my $storeidfield;
270
            if ( length($keepids) == 3 ) {
271
                $storeidfield = MARC::Field->new( $keepids, $originalid );
272
            } else {
273
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
274
            }
275
            $record->insert_fields_ordered($storeidfield);
276
            $record->delete_field( $record->field($tagid) );
232
    }
277
    }
233
	my $originalid;
278
    foreach my $stringfilter (@$filters) {
234
    if ($keepids){
279
        if ( length($stringfilter) == 3 ) {
235
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
280
            foreach my $field ( $record->field($stringfilter) ) {
236
      if ($originalid){
281
                $record->delete_field($field);
237
		 my $storeidfield;
282
                $debug && warn "removed : ", $field->as_string;
238
		 if (length($keepids)==3){
283
            }
239
		 	$storeidfield=MARC::Field->new($keepids,$originalid);
284
        } else {
240
		 }
285
            my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 )
241
		 else  {
286
              if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/;
242
			$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
287
            if ( ( $removetag > "010" ) && $removesubfield ) {
243
		 }
288
                foreach my $field ( $record->field($removetag) ) {
244
         $record->insert_fields_ordered($storeidfield);
289
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
245
	     $record->delete_field($record->field($tagid));
290
                    $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
246
      }
291
                }
292
            }
293
        }
247
    }
294
    }
248
    unless ($test_parameter) {
295
    unless ($test_parameter) {
249
        if ($authorities){
296
        if ($authorities){
250
            use C4::AuthoritiesMarc;
297
            use C4::AuthoritiesMarc;
251
            my $authtypecode=GuessAuthTypeCode($record);
298
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
252
            my $authid= ($id?$id:GuessAuthId($record));
299
            my $authid= ($id?$id:GuessAuthId($record));
253
            if ($authid && GetAuthority($authid)){
300
            if ($authid && GetAuthority($authid) && $update ){
254
            ## Authority has an id and is in database : Replace
301
            ## Authority has an id and is in database : Replace
255
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
302
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
256
                if ($@){
303
                if ($@){
Lines 283-288 RECORD: while ( ) { Link Here
283
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
330
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
284
				}
331
				}
285
 	        }
332
 	        }
333
            if ($yamlfile) {
334
            $yamlhash->{$originalid}->{'authid'} = $authid;
335
            my @subfields;
336
            foreach my $field ( $record->field("2..") ) {
337
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
338
            }
339
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
340
            }
286
        }
341
        }
287
        else {
342
        else {
288
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
343
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 306-325 RECORD: while ( ) { Link Here
306
			}
361
			}
307
					# create biblio, unless we already have it ( either match or isbn )
362
					# create biblio, unless we already have it ( either match or isbn )
308
            if ($biblionumber) {
363
            if ($biblionumber) {
309
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
364
                eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
310
			}
365
                if ($update) {
311
			else 
366
                    eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
312
			{
367
                    if ($@) {
313
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
368
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
369
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
370
                        next RECORD;
371
                    } else {
372
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
373
                    }
374
                } else {
375
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
376
                }
377
            } else {
378
                if ($insert) {
379
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
380
                    if ($@) {
381
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
382
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
383
                        next RECORD;
384
                    } else {
385
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
386
                    }
387
                } else {
388
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
389
                }
314
            }
390
            }
315
            if ( $@ ) {
316
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
317
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
318
                next RECORD;
319
            } 
320
 			else{
321
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
322
			}
323
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
391
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
324
            my $error_adding = $@;
392
            my $error_adding = $@;
325
            # Work on a clone so that if there are real errors, we can maybe
393
            # Work on a clone so that if there are real errors, we can maybe
Lines 384-389 RECORD: while ( ) { Link Here
384
            if ($#{ $errors_ref } > -1) {
452
            if ($#{ $errors_ref } > -1) {
385
                report_item_errors($biblionumber, $errors_ref);
453
                report_item_errors($biblionumber, $errors_ref);
386
            }
454
            }
455
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
387
        }
456
        }
388
        $dbh->commit() if (0 == $i % $commitnum);
457
        $dbh->commit() if (0 == $i % $commitnum);
389
    }
458
    }
Lines 407-412 if ($logfile){ Link Here
407
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
476
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
408
  $loghandle->close;
477
  $loghandle->close;
409
}
478
}
479
if ($yamlfile) {
480
    open YAML, "> $yamlfile" or die "cannot open $yamlfile \n";
481
    print YAML Dump($yamlhash);
482
}
410
exit 0;
483
exit 0;
411
484
412
sub GetRecordId{
485
sub GetRecordId{
Lines 462-470 sub report_item_errors { Link Here
462
}
535
}
463
sub printlog{
536
sub printlog{
464
	my $logelements=shift;
537
	my $logelements=shift;
465
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
538
	print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
539
}
540
sub get_heading_fields{
541
    my $headingfields;
542
    if ($authtypes){
543
        $headingfields=YAML::LoadFile($authtypes);
544
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
545
        $debug && warn YAML::Dump($headingfields);
546
    }
547
    unless ($headingfields){
548
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
549
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
550
    }
551
    return $headingfields;
466
}
552
}
467
468
553
469
=head1 NAME
554
=head1 NAME
470
555
Lines 526-532 The I<NUMBER> of records to wait before performing a 'commit' operation Link Here
526
611
527
File logs actions done for each record and their status into file
612
File logs actions done for each record and their status into file
528
613
529
=item B<-t>
614
=item B<-t, -test>
530
615
531
Test mode: parses the file, saying what he would do, but doing nothing.
616
Test mode: parses the file, saying what he would do, but doing nothing.
532
617
Lines 549-554 biblioitems, items Link Here
549
634
550
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
635
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
551
636
637
=item B<-authtypes>
638
639
file yamlfile with authoritiesTypes and distinguishable record field in order
640
to store the correct authtype
641
642
=item B<-yaml>
643
644
yaml file  format a yaml file with ids
645
keepids field store ids in field (usefull for authorities,
646
where 001 contains the authid for Koha, that can contain a very valuable
647
info for authorities coming from LOC or BNF. useless for biblios probably
648
649
=item B<-insert>
650
651
if set, only insert when possible
652
653
=item B<-update>
654
655
if set, only updates (any biblio should have a matching record)
656
657
=item B<-all>
658
659
if set, do whatever is required
660
552
=item B<-k, -keepids>=<FIELD>
661
=item B<-k, -keepids>=<FIELD>
553
662
554
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
663
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
555
- 

Return to bug 5635