View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-2 / +2 lines)
Lines 456-464 Get the record and tries to guess the adequate authtypecode from its content. Link Here
456
=cut
456
=cut
457
457
458
sub GuessAuthTypeCode {
458
sub GuessAuthTypeCode {
459
    my ($record) = @_;
459
    my ($record, $heading_fields) = @_;
460
    return unless defined $record;
460
    return unless defined $record;
461
my $heading_fields = {
461
    $heading_fields //= {
462
    "MARC21"=>{
462
    "MARC21"=>{
463
        '100'=>{authtypecode=>'PERSO_NAME'},
463
        '100'=>{authtypecode=>'PERSO_NAME'},
464
        '110'=>{authtypecode=>'CORPO_NAME'},
464
        '110'=>{authtypecode=>'CORPO_NAME'},
(-)a/misc/migration_tools/bulkmarcimport.pl (-51 / +167 lines)
Lines 24-38 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
33
binmode STDOUT, ':encoding(UTF-8)';
34
use open qw( :std :utf8 );
35
binmode( STDOUT, ":utf8" );
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
36
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
37
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
38
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
36
my $cleanisbn = 1;
39
my $cleanisbn = 1;
37
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
40
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
38
my $framework = '';
41
my $framework = '';
Lines 46-52 GetOptions( Link Here
46
    'o|offset:f' => \$offset,
49
    'o|offset:f' => \$offset,
47
    'h' => \$version,
50
    'h' => \$version,
48
    'd' => \$delete,
51
    'd' => \$delete,
49
    't' => \$test_parameter,
52
    't|test' => \$test_parameter,
50
    's' => \$skip_marc8_conversion,
53
    's' => \$skip_marc8_conversion,
51
    'c:s' => \$char_encoding,
54
    'c:s' => \$char_encoding,
52
    'v:s' => \$verbose,
55
    'v:s' => \$verbose,
Lines 56-77 GetOptions( Link Here
56
    'k|keepids:s' => \$keepids,
59
    'k|keepids:s' => \$keepids,
57
    'b|biblios' => \$biblios,
60
    'b|biblios' => \$biblios,
58
    'a|authorities' => \$authorities,
61
    'a|authorities' => \$authorities,
62
    'authtypes:s' => \$authtypes,
63
    'filter=s@'     => \$filters,
64
    'insert'        => \$insert,
65
    'update'        => \$update,
66
    'all'           => \$all,
59
    'match=s@'    => \$match,
67
    'match=s@'    => \$match,
60
    'i|isbn' => \$isbn_check,
68
    'i|isbn' => \$isbn_check,
61
    'x:s' => \$sourcetag,
69
    'x:s' => \$sourcetag,
62
    'y:s' => \$sourcesubfield,
70
    'y:s' => \$sourcesubfield,
63
    'idmap:s' => \$idmapfl,
71
    'idmap:s' => \$idmapfl,
64
    'cleanisbn!'     => \$cleanisbn,
72
    'cleanisbn!'     => \$cleanisbn,
73
    'yaml:s'        => \$yamlfile,
65
    'dedupbarcode' => \$dedup_barcode,
74
    'dedupbarcode' => \$dedup_barcode,
66
    'framework=s' => \$framework,
75
    'framework=s' => \$framework,
67
);
76
);
68
$biblios=!$authorities||$biblios;
77
$biblios ||= !$authorities;
78
$insert  ||= !$update;
79
80
if ($all) {
81
    $insert = 1;
82
    $update = 1;
83
}
69
84
70
if ($version || ($input_marc_file eq '')) {
85
if ($version || ($input_marc_file eq '')) {
71
    pod2usage( -verbose => 2 );
86
    pod2usage( -verbose => 2 );
72
    exit;
87
    exit;
73
}
88
}
74
89
90
my $dbh = C4::Context->dbh;
91
my $heading_fields=get_heading_fields();
92
75
if (defined $idmapfl) {
93
if (defined $idmapfl) {
76
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
94
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
77
}
95
}
Lines 81-88 if ((not defined $sourcesubfield) && (not defined $sourcetag)){ Link Here
81
  $sourcesubfield="a";
99
  $sourcesubfield="a";
82
}
100
}
83
101
84
my $dbh = C4::Context->dbh;
85
86
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & 
102
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & 
87
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
103
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
88
my $CataloguingLog = C4::Context->preference('CataloguingLog');
104
my $CataloguingLog = C4::Context->preference('CataloguingLog');
Lines 142-148 $batch->warnings_off(); Link Here
142
$batch->strict_off();
158
$batch->strict_off();
143
my $i=0;
159
my $i=0;
144
my $commitnum = $commit ? $commit : 50;
160
my $commitnum = $commit ? $commit : 50;
145
161
my $yamlhash;
146
162
147
# Skip file offset
163
# Skip file offset
148
if ( $offset ) {
164
if ( $offset ) {
Lines 199-204 RECORD: while ( ) { Link Here
199
            next RECORD;            
215
            next RECORD;            
200
        }
216
        }
201
    }
217
    }
218
    SetUTF8Flag($record);
202
    my $isbn;
219
    my $isbn;
203
    # remove trailing - in isbn (only for biblios, of course)
220
    # remove trailing - in isbn (only for biblios, of course)
204
    if ($biblios && $cleanisbn) {
221
    if ($biblios && $cleanisbn) {
Lines 212-256 RECORD: while ( ) { Link Here
212
    }
229
    }
213
    my $id;
230
    my $id;
214
    # search for duplicates (based on Local-number)
231
    # search for duplicates (based on Local-number)
215
    if ($match){
232
    my $originalid;
216
       require C4::Search;
233
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
217
       my $query=build_query($match,$record);
234
    if ($match) {
218
       my $server=($authorities?'authorityserver':'biblioserver');
235
        require C4::Search;
219
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
236
        my $query = build_query( $match, $record );
220
       die "unable to search the database for duplicates : $error" if (defined $error);
237
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
221
       #warn "$query $server : $totalhits";
238
        $debug && warn $query;
222
       if ( @{$results} == 1 ){
239
        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
223
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
240
        die "unable to search the database for duplicates : $error" if ( defined $error );
224
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
241
        $debug && warn "$query $server : $totalhits";
225
       } 
242
        if ( $results && scalar(@$results) == 1 ) {
226
       elsif  ( @{$results} > 1){
243
            my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
227
           $debug && warn "more than one match for $query";
244
            SetUTF8Flag($marcrecord);
228
       } 
245
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
229
       else {
246
            if ( $authorities && $marcFlavour ) {
230
           $debug && warn "nomatch for $query";
247
                #Skip if authority in database is the same as the on in database
231
       }
248
                if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
249
                    if ($yamlfile) {
250
                        $yamlhash->{$originalid}->{'authid'} = $id;
251
252
                        # we recover all subfields of the heading authorities
253
                        my @subfields;
254
                        foreach my $field ( $marcrecord->field("2..") ) {
255
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
256
                        }
257
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
258
                    }
259
                    next;
260
                }
261
            }
262
        } elsif ( $results && scalar(@$results) > 1 ) {
263
            $debug && warn "more than one match for $query";
264
        } else {
265
            $debug && warn "nomatch for $query";
266
        }
232
    }
267
    }
233
	my $originalid;
268
    if ($keepids && $originalid) {
234
    if ($keepids){
269
            my $storeidfield;
235
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
270
            if ( length($keepids) == 3 ) {
236
      if ($originalid){
271
                $storeidfield = MARC::Field->new( $keepids, $originalid );
237
		 my $storeidfield;
272
            } else {
238
		 if (length($keepids)==3){
273
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
239
		 	$storeidfield=MARC::Field->new($keepids,$originalid);
274
            }
240
		 }
275
            $record->insert_fields_ordered($storeidfield);
241
		 else  {
276
            $record->delete_field( $record->field($tagid) );
242
			$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
277
    }
243
		 }
278
    foreach my $stringfilter (@$filters) {
244
         $record->insert_fields_ordered($storeidfield);
279
        if ( length($stringfilter) == 3 ) {
245
	     $record->delete_field($record->field($tagid));
280
            foreach my $field ( $record->field($stringfilter) ) {
246
      }
281
                $record->delete_field($field);
282
                $debug && warn "removed : ", $field->as_string;
283
            }
284
        } else {
285
            my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 )
286
              if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/;
287
            if ( ( $removetag > "010" ) && $removesubfield ) {
288
                foreach my $field ( $record->field($removetag) ) {
289
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
290
                    $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
291
                }
292
            }
293
        }
247
    }
294
    }
248
    unless ($test_parameter) {
295
    unless ($test_parameter) {
249
        if ($authorities){
296
        if ($authorities){
250
            use C4::AuthoritiesMarc;
297
            use C4::AuthoritiesMarc;
251
            my $authtypecode=GuessAuthTypeCode($record);
298
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
252
            my $authid= ($id?$id:GuessAuthId($record));
299
            my $authid= ($id?$id:GuessAuthId($record));
253
            if ($authid && GetAuthority($authid)){
300
            if ($authid && GetAuthority($authid) && $update ){
254
            ## Authority has an id and is in database : Replace
301
            ## Authority has an id and is in database : Replace
255
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
302
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
256
                if ($@){
303
                if ($@){
Lines 283-288 RECORD: while ( ) { Link Here
283
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
330
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
284
				}
331
				}
285
 	        }
332
 	        }
333
            if ($yamlfile) {
334
            $yamlhash->{$originalid}->{'authid'} = $authid;
335
            my @subfields;
336
            foreach my $field ( $record->field("2..") ) {
337
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
338
            }
339
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
340
            }
286
        }
341
        }
287
        else {
342
        else {
288
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
343
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 306-325 RECORD: while ( ) { Link Here
306
			}
361
			}
307
					# create biblio, unless we already have it ( either match or isbn )
362
					# create biblio, unless we already have it ( either match or isbn )
308
            if ($biblionumber) {
363
            if ($biblionumber) {
364
<<<<<<< HEAD
309
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
365
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
310
			}
366
			}
311
			else 
367
			else 
312
			{
368
			{
313
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
369
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
370
=======
371
                eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
372
                if ($update) {
373
                    eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
374
                    if ($@) {
375
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
376
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
377
                        next RECORD;
378
                    } else {
379
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
380
                    }
381
                } else {
382
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
383
                }
384
            } else {
385
                if ($insert) {
386
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
387
                    if ($@) {
388
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
389
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
390
                        next RECORD;
391
                    } else {
392
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
393
                    }
394
                } else {
395
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
396
                }
397
>>>>>>> Bug 5635: bulkmarcimport new parameters & features
314
            }
398
            }
315
            if ( $@ ) {
316
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
317
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
318
                next RECORD;
319
            } 
320
 			else{
321
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
322
			}
323
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
399
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
324
            my $error_adding = $@;
400
            my $error_adding = $@;
325
            # Work on a clone so that if there are real errors, we can maybe
401
            # Work on a clone so that if there are real errors, we can maybe
Lines 384-389 RECORD: while ( ) { Link Here
384
            if ($#{ $errors_ref } > -1) {
460
            if ($#{ $errors_ref } > -1) {
385
                report_item_errors($biblionumber, $errors_ref);
461
                report_item_errors($biblionumber, $errors_ref);
386
            }
462
            }
463
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
387
        }
464
        }
388
        $dbh->commit() if (0 == $i % $commitnum);
465
        $dbh->commit() if (0 == $i % $commitnum);
389
    }
466
    }
Lines 407-412 if ($logfile){ Link Here
407
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
484
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
408
  $loghandle->close;
485
  $loghandle->close;
409
}
486
}
487
if ($yamlfile) {
488
    open YAML, "> $yamlfile" or die "cannot open $yamlfile \n";
489
    print YAML Dump($yamlhash);
490
}
410
exit 0;
491
exit 0;
411
492
412
sub GetRecordId{
493
sub GetRecordId{
Lines 462-470 sub report_item_errors { Link Here
462
}
543
}
463
sub printlog{
544
sub printlog{
464
	my $logelements=shift;
545
	my $logelements=shift;
465
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
546
	print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
547
}
548
sub get_heading_fields{
549
    my $headingfields;
550
    if ($authtypes){
551
        $headingfields=YAML::LoadFile($authtypes);
552
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
553
        $debug && warn YAML::Dump($headingfields);
554
    }
555
    unless ($headingfields){
556
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
557
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
558
    }
559
    return $headingfields;
466
}
560
}
467
468
561
469
=head1 NAME
562
=head1 NAME
470
563
Lines 526-532 The I<NUMBER> of records to wait before performing a 'commit' operation Link Here
526
619
527
File logs actions done for each record and their status into file
620
File logs actions done for each record and their status into file
528
621
529
=item B<-t>
622
=item B<-t, -test>
530
623
531
Test mode: parses the file, saying what he would do, but doing nothing.
624
Test mode: parses the file, saying what he would do, but doing nothing.
532
625
Lines 549-554 biblioitems, items Link Here
549
642
550
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
643
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
551
644
645
=item B<-authtypes>
646
647
file yamlfile with authoritiesTypes and distinguishable record field in order
648
to store the correct authtype
649
650
=item B<-yaml>
651
652
yaml file  format a yaml file with ids
653
keepids field store ids in field (usefull for authorities,
654
where 001 contains the authid for Koha, that can contain a very valuable
655
info for authorities coming from LOC or BNF. useless for biblios probably
656
657
=item B<-insert>
658
659
if set, only insert when possible
660
661
=item B<-update>
662
663
if set, only updates (any biblio should have a matching record)
664
665
=item B<-all>
666
667
if set, do whatever is required
668
552
=item B<-k, -keepids>=<FIELD>
669
=item B<-k, -keepids>=<FIELD>
553
670
554
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
671
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
555
- 

Return to bug 5635