View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/C4/AuthoritiesMarc.pm (-2 / +2 lines)
Lines 478-486 Get the record and tries to guess the adequate authtypecode from its content. Link Here
478
=cut
478
=cut
479
479
480
sub GuessAuthTypeCode {
480
sub GuessAuthTypeCode {
481
    my ($record) = @_;
481
    my ($record, $heading_fields) = @_;
482
    return unless defined $record;
482
    return unless defined $record;
483
my $heading_fields = {
483
    $heading_fields //= {
484
    "MARC21"=>{
484
    "MARC21"=>{
485
        '100'=>{authtypecode=>'PERSO_NAME'},
485
        '100'=>{authtypecode=>'PERSO_NAME'},
486
        '110'=>{authtypecode=>'CORPO_NAME'},
486
        '110'=>{authtypecode=>'CORPO_NAME'},
(-)a/misc/migration_tools/bulkmarcimport.pl (-59 / +172 lines)
Lines 24-38 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
33
binmode STDOUT, ':encoding(UTF-8)';
34
use open qw( :std :encoding(UTF-8) );
35
binmode( STDOUT, ":encoding(UTF-8)" );
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
36
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
37
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
38
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes );
36
my $cleanisbn = 1;
39
my $cleanisbn = 1;
37
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
40
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
38
my $framework = '';
41
my $framework = '';
Lines 46-52 GetOptions( Link Here
46
    'o|offset:f' => \$offset,
49
    'o|offset:f' => \$offset,
47
    'h' => \$version,
50
    'h' => \$version,
48
    'd' => \$delete,
51
    'd' => \$delete,
49
    't' => \$test_parameter,
52
    't|test' => \$test_parameter,
50
    's' => \$skip_marc8_conversion,
53
    's' => \$skip_marc8_conversion,
51
    'c:s' => \$char_encoding,
54
    'c:s' => \$char_encoding,
52
    'v:s' => \$verbose,
55
    'v:s' => \$verbose,
Lines 56-77 GetOptions( Link Here
56
    'k|keepids:s' => \$keepids,
59
    'k|keepids:s' => \$keepids,
57
    'b|biblios' => \$biblios,
60
    'b|biblios' => \$biblios,
58
    'a|authorities' => \$authorities,
61
    'a|authorities' => \$authorities,
62
    'authtypes:s' => \$authtypes,
63
    'filter=s@'     => \$filters,
64
    'insert'        => \$insert,
65
    'update'        => \$update,
66
    'all'           => \$all,
59
    'match=s@'    => \$match,
67
    'match=s@'    => \$match,
60
    'i|isbn' => \$isbn_check,
68
    'i|isbn' => \$isbn_check,
61
    'x:s' => \$sourcetag,
69
    'x:s' => \$sourcetag,
62
    'y:s' => \$sourcesubfield,
70
    'y:s' => \$sourcesubfield,
63
    'idmap:s' => \$idmapfl,
71
    'idmap:s' => \$idmapfl,
64
    'cleanisbn!'     => \$cleanisbn,
72
    'cleanisbn!'     => \$cleanisbn,
73
    'yaml:s'        => \$yamlfile,
65
    'dedupbarcode' => \$dedup_barcode,
74
    'dedupbarcode' => \$dedup_barcode,
66
    'framework=s' => \$framework,
75
    'framework=s' => \$framework,
67
);
76
);
68
$biblios=!$authorities||$biblios;
77
$biblios ||= !$authorities;
78
$insert  ||= !$update;
79
80
if ($all) {
81
    $insert = 1;
82
    $update = 1;
83
}
69
84
70
if ($version || ($input_marc_file eq '')) {
85
if ($version || ($input_marc_file eq '')) {
71
    pod2usage( -verbose => 2 );
86
    pod2usage( -verbose => 2 );
72
    exit;
87
    exit;
73
}
88
}
74
89
90
my $dbh = C4::Context->dbh;
91
my $heading_fields=get_heading_fields();
92
75
if (defined $idmapfl) {
93
if (defined $idmapfl) {
76
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
94
  open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
77
}
95
}
Lines 81-88 if ((not defined $sourcesubfield) && (not defined $sourcetag)){ Link Here
81
  $sourcesubfield="a";
99
  $sourcesubfield="a";
82
}
100
}
83
101
84
my $dbh = C4::Context->dbh;
85
86
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & 
102
# save the CataloguingLog property : we don't want to log a bulkmarcimport. It will slow the import & 
87
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
103
# will create problems in the action_logs table, that can't handle more than 1 entry per second per user.
88
my $CataloguingLog = C4::Context->preference('CataloguingLog');
104
my $CataloguingLog = C4::Context->preference('CataloguingLog');
Lines 142-148 $batch->warnings_off(); Link Here
142
$batch->strict_off();
158
$batch->strict_off();
143
my $i=0;
159
my $i=0;
144
my $commitnum = $commit ? $commit : 50;
160
my $commitnum = $commit ? $commit : 50;
145
161
my $yamlhash;
146
162
147
# Skip file offset
163
# Skip file offset
148
if ( $offset ) {
164
if ( $offset ) {
Lines 199-204 RECORD: while ( ) { Link Here
199
            next RECORD;            
215
            next RECORD;            
200
        }
216
        }
201
    }
217
    }
218
    SetUTF8Flag($record);
202
    my $isbn;
219
    my $isbn;
203
    # remove trailing - in isbn (only for biblios, of course)
220
    # remove trailing - in isbn (only for biblios, of course)
204
    if ($biblios && $cleanisbn) {
221
    if ($biblios && $cleanisbn) {
Lines 212-256 RECORD: while ( ) { Link Here
212
    }
229
    }
213
    my $id;
230
    my $id;
214
    # search for duplicates (based on Local-number)
231
    # search for duplicates (based on Local-number)
215
    if ($match){
232
    my $originalid;
216
       require C4::Search;
233
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
217
       my $query=build_query($match,$record);
234
    if ($match) {
218
       my $server=($authorities?'authorityserver':'biblioserver');
235
        require C4::Search;
219
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
236
        my $query = build_query( $match, $record );
220
       die "unable to search the database for duplicates : $error" if (defined $error);
237
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
221
       #warn "$query $server : $totalhits";
238
        $debug && warn $query;
222
       if ( @{$results} == 1 ){
239
        my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] );
223
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
240
        die "unable to search the database for duplicates : $error" if ( defined $error );
224
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
241
        $debug && warn "$query $server : $totalhits";
225
       } 
242
        if ( $results && scalar(@$results) == 1 ) {
226
       elsif  ( @{$results} > 1){
243
            my $marcrecord = MARC::File::USMARC::decode( $results->[0] );
227
           $debug && warn "more than one match for $query";
244
            SetUTF8Flag($marcrecord);
228
       } 
245
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
229
       else {
246
            if ( $authorities && $marcFlavour ) {
230
           $debug && warn "nomatch for $query";
247
                #Skip if authority in database is the same as the on in database
231
       }
248
                if ( $marcrecord->field('005')->data >= $record->field('005')->data ) {
249
                    if ($yamlfile) {
250
                        $yamlhash->{$originalid}->{'authid'} = $id;
251
252
                        # we recover all subfields of the heading authorities
253
                        my @subfields;
254
                        foreach my $field ( $marcrecord->field("2..") ) {
255
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
256
                        }
257
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
258
                    }
259
                    next;
260
                }
261
            }
262
        } elsif ( $results && scalar(@$results) > 1 ) {
263
            $debug && warn "more than one match for $query";
264
        } else {
265
            $debug && warn "nomatch for $query";
266
        }
267
    }
268
    if ($keepids && $originalid) {
269
            my $storeidfield;
270
            if ( length($keepids) == 3 ) {
271
                $storeidfield = MARC::Field->new( $keepids, $originalid );
272
            } else {
273
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
274
            }
275
            $record->insert_fields_ordered($storeidfield);
276
            $record->delete_field( $record->field($tagid) );
232
    }
277
    }
233
	my $originalid;
278
    foreach my $stringfilter (@$filters) {
234
    if ($keepids){
279
        if ( length($stringfilter) == 3 ) {
235
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
280
            foreach my $field ( $record->field($stringfilter) ) {
236
      if ($originalid){
281
                $record->delete_field($field);
237
		 my $storeidfield;
282
                $debug && warn "removed : ", $field->as_string;
238
		 if (length($keepids)==3){
283
            }
239
		 	$storeidfield=MARC::Field->new($keepids,$originalid);
284
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
240
		 }
285
            my $removetag = $1;
241
		 else  {
286
            my $removesubfield = $2;
242
			$storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid);
287
            my $removematch = $3;
243
		 }
288
            if ( ( $removetag > "010" ) && $removesubfield ) {
244
         $record->insert_fields_ordered($storeidfield);
289
                foreach my $field ( $record->field($removetag) ) {
245
	     $record->delete_field($record->field($tagid));
290
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
246
      }
291
                    $debug && warn "Potentially removed : ", $field->subfield($removesubfield);
292
                }
293
            }
294
        }
247
    }
295
    }
248
    unless ($test_parameter) {
296
    unless ($test_parameter) {
249
        if ($authorities){
297
        if ($authorities){
250
            use C4::AuthoritiesMarc;
298
            use C4::AuthoritiesMarc;
251
            my $authtypecode=GuessAuthTypeCode($record);
299
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
252
            my $authid= ($id?$id:GuessAuthId($record));
300
            my $authid= ($id?$id:GuessAuthId($record));
253
            if ($authid && GetAuthority($authid)){
301
            if ($authid && GetAuthority($authid) && $update ){
254
            ## Authority has an id and is in database : Replace
302
            ## Authority has an id and is in database : Replace
255
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
303
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
256
                if ($@){
304
                if ($@){
Lines 283-288 RECORD: while ( ) { Link Here
283
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
331
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
284
				}
332
				}
285
 	        }
333
 	        }
334
            if ($yamlfile) {
335
            $yamlhash->{$originalid}->{'authid'} = $authid;
336
            my @subfields;
337
            foreach my $field ( $record->field("2..") ) {
338
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
339
            }
340
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
341
            }
286
        }
342
        }
287
        else {
343
        else {
288
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
344
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 306-325 RECORD: while ( ) { Link Here
306
			}
362
			}
307
					# create biblio, unless we already have it ( either match or isbn )
363
					# create biblio, unless we already have it ( either match or isbn )
308
            if ($biblionumber) {
364
            if ($biblionumber) {
309
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
365
                eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
310
			}
366
                if ($update) {
311
			else 
367
                    eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) };
312
			{
368
                    if ($@) {
313
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
369
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
370
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
371
                        next RECORD;
372
                    } else {
373
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
374
                    }
375
                } else {
376
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
377
                }
378
            } else {
379
                if ($insert) {
380
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
381
                    if ($@) {
382
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
383
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
384
                        next RECORD;
385
                    } else {
386
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
387
                    }
388
                } else {
389
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
390
                }
314
            }
391
            }
315
            if ( $@ ) {
316
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
317
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
318
                next RECORD;
319
            } 
320
 			else{
321
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
322
			}
323
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
392
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
324
            my $error_adding = $@;
393
            my $error_adding = $@;
325
            # Work on a clone so that if there are real errors, we can maybe
394
            # Work on a clone so that if there are real errors, we can maybe
Lines 384-389 RECORD: while ( ) { Link Here
384
            if ($#{ $errors_ref } > -1) {
453
            if ($#{ $errors_ref } > -1) {
385
                report_item_errors($biblionumber, $errors_ref);
454
                report_item_errors($biblionumber, $errors_ref);
386
            }
455
            }
456
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
387
        }
457
        }
388
        $dbh->commit() if (0 == $i % $commitnum);
458
        $dbh->commit() if (0 == $i % $commitnum);
389
    }
459
    }
Lines 407-412 if ($logfile){ Link Here
407
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
477
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
408
  $loghandle->close;
478
  $loghandle->close;
409
}
479
}
480
if ($yamlfile) {
481
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
482
    print $yamlfileout Dump($yamlhash);
483
}
410
exit 0;
484
exit 0;
411
485
412
sub GetRecordId{
486
sub GetRecordId{
Lines 445-458 sub build_query { Link Here
445
sub build_simplequery {
519
sub build_simplequery {
446
	my $element=shift;
520
	my $element=shift;
447
	my $record=shift;
521
	my $record=shift;
448
        my ($index,$recorddata)=split /,/,$element;
522
    my @searchstrings;
449
        my ($tag,$subfields) =($1,$2) if ($recorddata=~/(\d{3})(.*)/);
523
    my ($index,$recorddata)=split /,/,$element;
450
        my @searchstrings;
524
    if ($recorddata=~/(\d{3})(.*)/) {
525
        my ($tag,$subfields) =($1,$2);
451
        foreach my $field ($record->field($tag)){
526
        foreach my $field ($record->field($tag)){
452
		  if (length($field->as_string("$subfields"))>0){
527
		  if (length($field->as_string("$subfields"))>0){
453
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
528
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
454
		  }
529
		  }
455
        }
530
        }
531
    }
456
    my $QParser;
532
    my $QParser;
457
    $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
533
    $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
458
    my $op;
534
    my $op;
Lines 478-486 sub report_item_errors { Link Here
478
}
554
}
479
sub printlog{
555
sub printlog{
480
	my $logelements=shift;
556
	my $logelements=shift;
481
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
557
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
558
}
559
sub get_heading_fields{
560
    my $headingfields;
561
    if ($authtypes){
562
        $headingfields=YAML::LoadFile($authtypes);
563
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
564
        $debug && warn YAML::Dump($headingfields);
565
    }
566
    unless ($headingfields){
567
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
568
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
569
    }
570
    return $headingfields;
482
}
571
}
483
484
572
485
=head1 NAME
573
=head1 NAME
486
574
Lines 542-548 The I<NUMBER> of records to wait before performing a 'commit' operation Link Here
542
630
543
File logs actions done for each record and their status into file
631
File logs actions done for each record and their status into file
544
632
545
=item B<-t>
633
=item B<-t, -test>
546
634
547
Test mode: parses the file, saying what he would do, but doing nothing.
635
Test mode: parses the file, saying what he would do, but doing nothing.
548
636
Lines 565-570 biblioitems, items Link Here
565
653
566
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
654
Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709)
567
655
656
=item B<-authtypes>
657
658
file yamlfile with authoritiesTypes and distinguishable record field in order
659
to store the correct authtype
660
661
=item B<-yaml>
662
663
yaml file  format a yaml file with ids
664
665
=item B<-filter>
666
667
list of fields that will not be imported. Can be any from 000 to 999 or field,
668
subfield and subfield's matching value such as 200avalue
669
670
=item B<-insert>
671
672
if set, only insert when possible
673
674
=item B<-update>
675
676
if set, only updates (any biblio should have a matching record)
677
678
=item B<-all>
679
680
if set, do whatever is required
681
568
=item B<-k, -keepids>=<FIELD>
682
=item B<-k, -keepids>=<FIELD>
569
683
570
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
684
Field store ids in I<FIELD> (usefull for authorities, where 001 contains the
571
- 

Return to bug 5635