View | Details | Raw Unified | Return to bug 5635
Collapse All | Expand All

(-)a/misc/migration_tools/bulkmarcimport.pl (-38 / +169 lines)
Lines 24-39 use C4::Koha; Link Here
24
use C4::Debug;
24
use C4::Debug;
25
use C4::Charset;
25
use C4::Charset;
26
use C4::Items;
26
use C4::Items;
27
use YAML;
27
use Unicode::Normalize;
28
use Unicode::Normalize;
28
use Time::HiRes qw(gettimeofday);
29
use Time::HiRes qw(gettimeofday);
29
use Getopt::Long;
30
use Getopt::Long;
30
use IO::File;
31
use IO::File;
31
use Pod::Usage;
32
use Pod::Usage;
32
33
34
use open qw( :std :utf8 );
33
binmode(STDOUT, ":utf8");
35
binmode(STDOUT, ":utf8");
36
34
my ( $input_marc_file, $number, $offset) = ('',0,0);
37
my ( $input_marc_file, $number, $offset) = ('',0,0);
35
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
38
my ($version, $delete, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile,$yamlfile);
36
my ($sourcetag,$sourcesubfield,$idmapfl);
39
my ($sourcetag,$sourcesubfield,$idmapfl);
40
my ($insert,$filters,$update,$all,$test_parameter);
37
41
38
$|=1;
42
$|=1;
39
43
Lines 44-50 GetOptions( Link Here
44
    'o|offset:f' => \$offset,
48
    'o|offset:f' => \$offset,
45
    'h' => \$version,
49
    'h' => \$version,
46
    'd' => \$delete,
50
    'd' => \$delete,
47
    't' => \$test_parameter,
51
    't|test' => \$test_parameter,
48
    's' => \$skip_marc8_conversion,
52
    's' => \$skip_marc8_conversion,
49
    'c:s' => \$char_encoding,
53
    'c:s' => \$char_encoding,
50
    'v:s' => \$verbose,
54
    'v:s' => \$verbose,
Lines 54-70 GetOptions( Link Here
54
    'k|keepids:s' => \$keepids,
58
    'k|keepids:s' => \$keepids,
55
    'b|biblios' => \$biblios,
59
    'b|biblios' => \$biblios,
56
    'a|authorities' => \$authorities,
60
    'a|authorities' => \$authorities,
61
    'filter=s@' => \$filters,
62
    'insert' => \$insert,
63
    'update' => \$update,
64
    'all' => \$all,
57
    'match=s@'    => \$match,
65
    'match=s@'    => \$match,
58
    'i|isbn' => \$isbn_check,
66
    'i|isbn' => \$isbn_check,
59
    'x:s' => \$sourcetag,
67
    'x:s' => \$sourcetag,
60
    'y:s' => \$sourcesubfield,
68
    'y:s' => \$sourcesubfield,
61
    'idmap:s' => \$idmapfl,
69
    'idmap:s' => \$idmapfl,
70
    'yaml:s' => \$yamlfile,
62
);
71
);
63
$biblios=!$authorities||$biblios;
72
$biblios||= !$authorities;
73
$insert ||= !$update;
74
if ($all){
75
    $insert=1;
76
    $update=1;
77
}
64
78
65
if ($version || ($input_marc_file eq '')) {
79
if ($version || ($input_marc_file eq '')) {
66
    pod2usage( -verbose => 2 );
80
    print <<EOF ;
67
    exit;
81
Small script to import bibliographic records into Koha.
82
83
Parameters:
84
  h      this version/help screen
85
  file   /path/to/file/to/dump: the file to import
86
  v      verbose mode. 1 means "some infos", 2 means "MARC dumping"
87
  fk     Turn off foreign key checks during import.
88
  n      the number of records to import. If missing, all the file is imported
89
  o      file offset before importing, ie number of records to skip.
90
  commit the number of records to wait before performing a 'commit' operation
91
  l file logs actions done for each record and their status into file
92
  t      test mode: parses the file, saying what he would do, but doing nothing.
93
  s      skip automatic conversion of MARC-8 to UTF-8.  This option is 
94
         provided for debugging.
95
  c      the characteristic MARC flavour. At the moment, only MARC21 and 
96
         UNIMARC are supported. MARC21 by default.
97
  d      delete EVERYTHING related to biblio in koha-DB before import. Tables:
98
         biblio, biblioitems, titems
99
  m      format, MARCXML or ISO2709 (defaults to ISO2709)
100
  yaml file  format a yaml file with ids
101
  keepids field store ids in field (usefull for authorities, where 001 contains the authid for Koha, that can contain a very valuable info for authorities coming from LOC or BNF. useless for biblios probably)
102
  x      source bib tag for reporting the source bib number
103
  y      source subfield for reporting the source bib number
104
  idmap  file for the koha bib and source id
105
  keepids store ids in 009 (usefull for authorities, where 001 contains the authid for Koha, that can contain a very valuable info for authorities coming from LOC or BNF. useless for biblios probably)
106
  b|biblios type of import : bibliographic records
107
  a|authorities type of import : authority records
108
  match  matchindex,fieldtomatch matchpoint to use to deduplicate
109
          fieldtomatch can be either 001 to 999 
110
                       or field and list of subfields as such 100abcde
111
  test   if set, test mode only, donot add anything in database
112
  insert if set, only insert when possible
113
  update if set, only updates (any biblio should have a matching record)
114
  all    if set, do whatever is required
115
  i|isbn if set, a search will be done on isbn, and, if the same isbn is found, the biblio is not added. It's another
116
         method to deduplicate. 
117
         match & i can be both set.
118
IMPORTANT: don't use this script before you've entered and checked your MARC 
119
           parameters tables twice (or more!). Otherwise, the import won't work 
120
           correctly and you will get invalid data.
121
122
SAMPLE: 
123
  \$ export KOHA_CONF=/etc/koha.conf
124
  \$ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\
125
    -file /home/jmf/koha.mrc -n 3000
126
EOF
127
exit;
68
}
128
}
69
129
70
if (defined $idmapfl) {
130
if (defined $idmapfl) {
Lines 137-143 $batch->warnings_off(); Link Here
137
$batch->strict_off();
197
$batch->strict_off();
138
my $i=0;
198
my $i=0;
139
my $commitnum = $commit ? $commit : 50;
199
my $commitnum = $commit ? $commit : 50;
140
200
my $yamlhash;
141
201
142
# Skip file offset
202
# Skip file offset
143
if ( $offset ) {
203
if ( $offset ) {
Lines 195-208 RECORD: while ( ) { Link Here
195
            next RECORD;            
255
            next RECORD;            
196
        }
256
        }
197
    }
257
    }
258
    SetUTF8Flag($record);
198
    my $isbn;
259
    my $isbn;
199
    # remove trailing - in isbn (only for biblios, of course)
260
    # remove trailing - in isbn (only for biblios, of course)
200
    if ($biblios) {
261
    if ($biblios) {
201
        if ($marcFlavour eq 'UNIMARC') {
262
        if ($marcFlavour eq 'UNIMARC') {
202
            if (my $f010 = $record->field('010')) {
263
            if (my $f010 = $record->field('010')) {
203
                $isbn = $f010->subfield('a');
264
                if ($f010->subfield('a')){
204
                $isbn =~ s/-//g;
265
                    $isbn = $f010->subfield('a');
205
                $f010->update('a' => $isbn);
266
                    $isbn =~ s/-//g;
267
                    $f010->update('a' => $isbn);
268
                }
206
            }
269
            }
207
        } else {
270
        } else {
208
            if (my $f020 = $record->field('020')) {
271
            if (my $f020 = $record->field('020')) {
Lines 215-241 RECORD: while ( ) { Link Here
215
    }
278
    }
216
    my $id;
279
    my $id;
217
    # search for duplicates (based on Local-number)
280
    # search for duplicates (based on Local-number)
281
    my $originalid;
282
    $originalid=GetRecordId($record,$tagid,$subfieldid);
218
    if ($match){
283
    if ($match){
219
       require C4::Search;
284
       require C4::Search;
220
       my $query=build_query($match,$record);
285
       my $query=build_query($match,$record);
221
       my $server=($authorities?'authorityserver':'biblioserver');
286
       my $server=($authorities?'authorityserver':'biblioserver');
287
       $debug && warn $query;
222
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
288
       my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] );
223
       die "unable to search the database for duplicates : $error" if (defined $error);
289
       die "unable to search the database for duplicates : $error" if (defined $error);
224
       #warn "$query $server : $totalhits";
290
       $debug && warn "$query $server : $totalhits";
225
       if ($results && scalar(@$results)==1){
291
       if ($results && scalar(@$results)==1){
226
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
292
           my $marcrecord = MARC::File::USMARC::decode($results->[0]);
293
           SetUTF8Flag($marcrecord);
227
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
294
	   	   $id=GetRecordId($marcrecord,$tagid,$subfieldid);
228
       } 
295
           if ($authorities && $marcFlavour ) {
296
                #Skip if authority in database is the same as the on in database
297
				if ($marcrecord->field('005')->data >= $record->field('005')->data){
298
					if ($yamlfile){
299
						$yamlhash->{$originalid}->{'authid'}=$id;
300
						# On récupère tous les souschamps des champs vedettes d'autorités
301
						my @subfields;
302
						foreach my $field ($marcrecord->field("2..")){
303
						    push @subfields, map{
304
									($_->[0]=~/[a-z]/?$_->[1]:())
305
						    		       }  $field->subfields();
306
						}
307
						$yamlhash->{$originalid}->{'subfields'}=\@subfields;
308
					}
309
					next;
310
				}
311
			}
312
       }
229
       elsif  ($results && scalar(@$results)>1){
313
       elsif  ($results && scalar(@$results)>1){
230
       $debug && warn "more than one match for $query";
314
          $debug && warn "more than one match for $query";
231
       } 
315
       }
232
       else {
316
       else {
233
       $debug && warn "nomatch for $query";
317
          $debug && warn "nomatch for $query";
234
       }
318
       }
235
    }
319
    }
236
	my $originalid;
237
    if ($keepids){
320
    if ($keepids){
238
	  $originalid=GetRecordId($record,$tagid,$subfieldid);
239
      if ($originalid){
321
      if ($originalid){
240
		 my $storeidfield;
322
		 my $storeidfield;
241
		 if (length($keepids)==3){
323
		 if (length($keepids)==3){
Lines 248-261 RECORD: while ( ) { Link Here
248
	     $record->delete_field($record->field($tagid));
330
	     $record->delete_field($record->field($tagid));
249
      }
331
      }
250
    }
332
    }
251
    unless ($test_parameter) {
333
    foreach my $stringfilter (@$filters){
334
        if (length($stringfilter)==3){
335
            foreach my $field ($record->field($stringfilter)){
336
                $record->delete_field($field);
337
                $debug && warn "removed : ",$field->as_string;
338
            }
339
        }
340
        else {
341
                my ($removetag,$removesubfield,$removematch)=($1,$2,$3) 
342
                    if $stringfilter=~/([0-9]{3})([a-z0-9])(.*)/;
343
                if (($removetag >"010")&& $removesubfield){
344
                    foreach my $field ($record->field($removetag)){
345
                        $field->delete_subfield(code=>"$removesubfield",match=>$removematch);
346
                        $debug && warn "Potentially removed : ",$field->subfield($removesubfield);
347
                    }
348
                }
349
        }
350
    }
252
        if ($authorities){
351
        if ($authorities){
253
            use C4::AuthoritiesMarc;
352
            use C4::AuthoritiesMarc;
254
            my $authtypecode=GuessAuthTypeCode($record);
353
            my $authtypecode=GuessAuthTypeCode($record);
255
            my $authid= ($id?$id:GuessAuthId($record));
354
            my $authid= ($id?$id:GuessAuthId($record));
256
            if ($authid && GetAuthority($authid)){
355
            if ($authid && GetAuthority($authid) && $update){
257
            ## Authority has an id and is in database : Replace
356
            ## Authority has an id and is in database : Replace
258
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
357
                (! $test_parameter) and eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
259
                if ($@){
358
                if ($@){
260
                    warn "Problem with authority $authid Cannot Modify";
359
                    warn "Problem with authority $authid Cannot Modify";
261
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
360
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
Lines 266-272 RECORD: while ( ) { Link Here
266
            }  
365
            }  
267
            elsif (defined $authid) {
366
            elsif (defined $authid) {
268
            ## An authid is defined but no authority in database : add
367
            ## An authid is defined but no authority in database : add
269
                eval { ( $authid ) = AddAuthority($record,$authid, $authtypecode) };
368
                (! $test_parameter) and eval { ( $authid ) = AddAuthority($record,$authid, $authtypecode) };
270
                if ($@){
369
                if ($@){
271
                    warn "Problem with authority $authid Cannot Add ".$@;
370
                    warn "Problem with authority $authid Cannot Add ".$@;
272
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
371
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
Lines 277-283 RECORD: while ( ) { Link Here
277
            }
376
            }
278
	        else {
377
	        else {
279
            ## True insert in database
378
            ## True insert in database
280
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
379
                (! $test_parameter) and eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
281
                if ($@){
380
                if ($@){
282
                    warn "Problem with authority $authid Cannot Add".$@;
381
                    warn "Problem with authority $authid Cannot Add".$@;
283
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
382
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
Lines 286-291 RECORD: while ( ) { Link Here
286
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
385
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
287
				}
386
				}
288
 	        }
387
 	        }
388
	        if ($yamlfile){
389
              	$yamlhash->{$originalid}->{'authid'}=$authid;
390
				my @subfields;
391
				foreach my $field ($record->field("2..")){
392
		    		push @subfields, map{
393
							($_->[0]=~/[a-z]/?$_->[1]:())
394
		    		       	}  $field->subfields();
395
				}
396
	      		$yamlhash->{$originalid}->{'subfields'}=\@subfields;
397
            }
289
        }
398
        }
290
        else {
399
        else {
291
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
400
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
Lines 309-329 RECORD: while ( ) { Link Here
309
			}
418
			}
310
					# create biblio, unless we already have it ( either match or isbn )
419
					# create biblio, unless we already have it ( either match or isbn )
311
            if ($biblionumber) {
420
            if ($biblionumber) {
312
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}
421
				eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
422
                if ($update) {
423
                    (! $test_parameter) and eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio($record, $biblionumber,GetFrameworkcode($biblionumber)) };
424
                    if ( $@ ) {
425
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
426
                        printlog({id=>$id||$originalid||$biblionumber, op=>"update",status=>"ERROR"}) if ($logfile);
427
                        next RECORD;
428
                    }
429
                    else{
430
                        printlog({id=>$id||$originalid||$biblionumber, op=>"update",status=>"ok"}) if ($logfile);
431
                    }
432
                }
433
                else {
434
                   printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"warning : already in database"}) if ($logfile);
435
                }
313
			}
436
			}
314
			else 
437
			else 
315
			{
438
			{
316
                eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) };
439
                if ($insert){
440
                    (! $test_parameter) and eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) };
441
                    if ( $@ ) {
442
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
443
                        printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
444
                        next RECORD;
445
                    }
446
                    else{
447
                        printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
448
                    }
449
                }
450
                else {
451
                   printlog({id=>$id||$originalid||$biblionumber, op=>"update",status=>"warning : not in database"}) if ($logfile);
452
                }
317
            }
453
            }
318
            if ( $@ ) {
454
            (! $test_parameter) and eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
319
                warn "ERROR: Adding biblio $biblionumber failed: $@\n";
320
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile);
321
                next RECORD;
322
            } 
323
 			else{
324
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
325
			}
326
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
327
            if ( $@ ) {
455
            if ( $@ ) {
328
                warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
456
                warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
329
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
457
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
Lines 331-346 RECORD: while ( ) { Link Here
331
                # the MARC columns in biblioitems were not set.
459
                # the MARC columns in biblioitems were not set.
332
                ModBiblioMarc( $record, $biblionumber, '' );
460
                ModBiblioMarc( $record, $biblionumber, '' );
333
                next RECORD;
461
                next RECORD;
334
            } 
462
            }
335
 			else{
463
 			else{
336
				printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
464
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
337
			}
465
			}
338
            if ($#{ $errors_ref } > -1) { 
466
            if ($#{ $errors_ref } > -1) { 
339
                report_item_errors($biblionumber, $errors_ref);
467
                report_item_errors($biblionumber, $errors_ref);
340
            }
468
            }
469
            $yamlhash->{$originalid}=$biblionumber if ($yamlfile);
341
        }
470
        }
342
        $dbh->commit() if (0 == $i % $commitnum);
471
        $dbh->commit() if (0 == $i % $commitnum);
343
    }
344
    last if $i == $number;
472
    last if $i == $number;
345
}
473
}
346
$dbh->commit();
474
$dbh->commit();
Lines 361-366 if ($logfile){ Link Here
361
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
489
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
362
  $loghandle->close;
490
  $loghandle->close;
363
}
491
}
492
if ($yamlfile){
493
    open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; 
494
    print YAML Dump($yamlhash);
495
}
364
exit 0;
496
exit 0;
365
497
366
sub GetRecordId{
498
sub GetRecordId{
Lines 370-376 sub GetRecordId{ Link Here
370
	my $id;
502
	my $id;
371
	if ($tag lt "010"){
503
	if ($tag lt "010"){
372
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
504
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
373
	} 
505
	}
374
	elsif ($subfield){
506
	elsif ($subfield){
375
		if ($marcrecord->field($tag)){
507
		if ($marcrecord->field($tag)){
376
			return $marcrecord->subfield($tag,$subfield);
508
			return $marcrecord->subfield($tag,$subfield);
Lines 415-421 sub report_item_errors { Link Here
415
}
547
}
416
sub printlog{
548
sub printlog{
417
	my $logelements=shift;
549
	my $logelements=shift;
418
	print $loghandle join (";",@$logelements{qw<id op status>}),"\n";
550
	print $loghandle join (";",map{defined $_?$_:""}@$logelements{qw<id op status>}),"\n";
419
}
551
}
420
552
421
553
422
- 

Return to bug 5635