Bugzilla – Attachment 11071 Details for
Bug 5635
bulkmarcimport new parameters & features
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
proposed patch
0001-Bug-5635-bulkmarcimport-new-parameters-features.patch (text/plain), 14.11 KB, created by
Stéphane Delaune
on 2012-07-23 08:54:47 UTC
(
hide
)
Description:
proposed patch
Filename:
MIME Type:
Creator:
Stéphane Delaune
Created:
2012-07-23 08:54:47 UTC
Size:
14.11 KB
patch
obsolete
>From b1de98aea9d2c9a8155fb8098afb3969a7254d6b Mon Sep 17 00:00:00 2001 >From: =?UTF-8?q?St=C3=A9phane=20Delaune?= <stephane.delaune@biblibre.com> >Date: Mon, 23 Jul 2012 10:49:31 +0200 >Subject: [PATCH] Bug 5635: bulkmarcimport new parameters & features > >--- > C4/AuthoritiesMarc.pm | 4 +- > misc/migration_tools/bulkmarcimport.pl | 216 ++++++++++++++++++++++++-------- > 2 files changed, 165 insertions(+), 55 deletions(-) > >diff --git a/C4/AuthoritiesMarc.pm b/C4/AuthoritiesMarc.pm >index ece46a3..63d7f96 100644 >--- a/C4/AuthoritiesMarc.pm >+++ b/C4/AuthoritiesMarc.pm >@@ -447,9 +447,9 @@ Get the record and tries to guess the adequate authtypecode from its content. > =cut > > sub GuessAuthTypeCode { >- my ($record) = @_; >+ my ($record, $heading_fields) = @_; > return unless defined $record; >-my $heading_fields = { >+ $heading_fields //= { > "MARC21"=>{ > '100'=>{authtypecode=>'PERSO_NAME'}, > '110'=>{authtypecode=>'CORPO_NAME'}, >diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl >index a9fee95..d83af39 100755 >--- a/misc/migration_tools/bulkmarcimport.pl >+++ b/misc/migration_tools/bulkmarcimport.pl >@@ -24,15 +24,18 @@ use C4::Koha; > use C4::Debug; > use C4::Charset; > use C4::Items; >+use YAML; > use Unicode::Normalize; > use Time::HiRes qw(gettimeofday); > use Getopt::Long; > use IO::File; > use Pod::Usage; > >-binmode STDOUT, ':encoding(UTF-8)'; >+use open qw( :std :utf8 ); >+binmode( STDOUT, ":utf8" ); > my ( $input_marc_file, $number, $offset) = ('',0,0); > my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); >+my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); > my $cleanisbn = 1; > my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); > >@@ -45,7 +48,7 @@ GetOptions( > 'o|offset:f' => \$offset, > 'h' => \$version, > 'd' => \$delete, >- 't' => \$test_parameter, >+ 't|test' => \$test_parameter, > 's' => \$skip_marc8_conversion, > 'c:s' => \$char_encoding, > 'v:s' => \$verbose, >@@ -55,21 +58,35 @@ GetOptions( > 'k|keepids:s' => \$keepids, > 'b|biblios' => \$biblios, > 'a|authorities' => \$authorities, >+ 'authtypes:s' => \$authtypes, >+ 'filter=s@' => \$filters, >+ 'insert' => \$insert, >+ 'update' => \$update, >+ 'all' => \$all, > 'match=s@' => \$match, > 'i|isbn' => \$isbn_check, > 'x:s' => \$sourcetag, > 'y:s' => \$sourcesubfield, > 'idmap:s' => \$idmapfl, > 'cleanisbn!' => \$cleanisbn, >+ 'yaml:s' => \$yamlfile, > 'dedupbarcode' => \$dedup_barcode, > ); >-$biblios=!$authorities||$biblios; >+$biblios ||= !$authorities; >+$insert ||= !$update; >+ >+if ($all) { >+ $insert = 1; >+ $update = 1; >+} > > if ($version || ($input_marc_file eq '')) { > pod2usage( -verbose => 2 ); > exit; > } > >+my $heading_fields=get_heading_fields(); >+ > if (defined $idmapfl) { > open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; > } >@@ -140,7 +157,7 @@ $batch->warnings_off(); > $batch->strict_off(); > my $i=0; > my $commitnum = $commit ? $commit : 50; >- >+my $yamlhash; > > # Skip file offset > if ( $offset ) { >@@ -197,6 +214,7 @@ RECORD: while ( ) { > next RECORD; > } > } >+ SetUTF8Flag($record); > my $isbn; > # remove trailing - in isbn (only for biblios, of course) > if ($biblios && $cleanisbn) { >@@ -210,45 +228,75 @@ RECORD: while ( ) { > } > my $id; > # search for duplicates (based on Local-number) >- if ($match){ >- require C4::Search; >- my $query=build_query($match,$record); >- my $server=($authorities?'authorityserver':'biblioserver'); >- my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); >- die "unable to search the database for duplicates : $error" if (defined $error); >- #warn "$query $server : $totalhits"; >- if ( @{$results} == 1 ){ >- my $marcrecord = MARC::File::USMARC::decode($results->[0]); >- $id=GetRecordId($marcrecord,$tagid,$subfieldid); >- } >- elsif ( @{$results} > 1){ >- $debug && warn "more than one match for $query"; >- } >- else { >- $debug && warn "nomatch for $query"; >- } >+ my $originalid; >+ $originalid = GetRecordId( $record, $tagid, $subfieldid ); >+ if ($match) { >+ require C4::Search; >+ my $query = build_query( $match, $record ); >+ my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); >+ $debug && warn $query; >+ my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); >+ die "unable to search the database for duplicates : $error" if ( defined $error ); >+ $debug && warn "$query $server : $totalhits"; >+ if ( $results && scalar(@$results) == 1 ) { >+ my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); >+ SetUTF8Flag($marcrecord); >+ $id = GetRecordId( $marcrecord, $tagid, $subfieldid ); >+ if ( $authorities && $marcFlavour ) { >+ #Skip if authority in database is the same as the on in database >+ if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { >+ if ($yamlfile) { >+ $yamlhash->{$originalid}->{'authid'} = $id; >+ >+ # On récupère tous les souschamps des champs vedettes d'autorités >+ my @subfields; >+ foreach my $field ( $marcrecord->field("2..") ) { >+ push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); >+ } >+ $yamlhash->{$originalid}->{'subfields'} = \@subfields; >+ } >+ next; >+ } >+ } >+ } elsif ( $results && scalar(@$results) > 1 ) { >+ $debug && warn "more than one match for $query"; >+ } else { >+ $debug && warn "nomatch for $query"; >+ } > } >- my $originalid; >- if ($keepids){ >- $originalid=GetRecordId($record,$tagid,$subfieldid); >- if ($originalid){ >- my $storeidfield; >- if (length($keepids)==3){ >- $storeidfield=MARC::Field->new($keepids,$originalid); >- } >- else { >- $storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); >- } >- $record->insert_fields_ordered($storeidfield); >- $record->delete_field($record->field($tagid)); >- } >+ if ($keepids && $originalid) { >+ my $storeidfield; >+ if ( length($keepids) == 3 ) { >+ $storeidfield = MARC::Field->new( $keepids, $originalid ); >+ } else { >+ $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); >+ } >+ $record->insert_fields_ordered($storeidfield); >+ $record->delete_field( $record->field($tagid) ); >+ } >+ foreach my $stringfilter (@$filters) { >+ if ( length($stringfilter) == 3 ) { >+ foreach my $field ( $record->field($stringfilter) ) { >+ $record->delete_field($field); >+ $debug && warn "removed : ", $field->as_string; >+ } >+ } else { >+ my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) >+ if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; >+ if ( ( $removetag > "010" ) && $removesubfield ) { >+ foreach my $field ( $record->field($removetag) ) { >+ $field->delete_subfield( code => "$removesubfield", match => $removematch ); >+ $debug && warn "Potentially removed : ", $field->subfield($removesubfield); >+ } >+ } >+ } > } > unless ($test_parameter) { > if ($authorities){ > use C4::AuthoritiesMarc; >- my $authtypecode=GuessAuthTypeCode($record); >+ my $authtypecode=GuessAuthTypeCode($record, $heading_fields); > my $authid= ($id?$id:GuessAuthId($record)); >- if ($authid && GetAuthority($authid)){ >+ if ($authid && GetAuthority($authid) && $update ){ > ## Authority has an id and is in database : Replace > eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; > if ($@){ >@@ -281,6 +329,14 @@ RECORD: while ( ) { > printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); > } > } >+ if ($yamlfile) { >+ $yamlhash->{$originalid}->{'authid'} = $authid; >+ my @subfields; >+ foreach my $field ( $record->field("2..") ) { >+ push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); >+ } >+ $yamlhash->{$originalid}->{'subfields'} = \@subfields; >+ } > } > else { > my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); >@@ -304,20 +360,33 @@ RECORD: while ( ) { > } > # create biblio, unless we already have it ( either match or isbn ) > if ($biblionumber) { >- eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} >- } >- else >- { >- eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) }; >+ eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; >+ if ($update) { >+ eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; >+ if ($@) { >+ warn "ERROR: Edit biblio $biblionumber failed: $@\n"; >+ printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); >+ next RECORD; >+ } else { >+ printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); >+ } >+ } else { >+ printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); >+ } >+ } else { >+ if ($insert) { >+ eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; >+ if ($@) { >+ warn "ERROR: Adding biblio $biblionumber failed: $@\n"; >+ printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); >+ next RECORD; >+ } else { >+ printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); >+ } >+ } else { >+ printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); >+ } > } >- if ( $@ ) { >- warn "ERROR: Adding biblio $biblionumber failed: $@\n"; >- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); >- next RECORD; >- } >- else{ >- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); >- } > eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; > my $error_adding = $@; > # Work on a clone so that if there are real errors, we can maybe >@@ -382,6 +451,7 @@ RECORD: while ( ) { > if ($#{ $errors_ref } > -1) { > report_item_errors($biblionumber, $errors_ref); > } >+ $yamlhash->{$originalid} = $biblionumber if ($yamlfile); > } > $dbh->commit() if (0 == $i % $commitnum); > } >@@ -405,6 +475,10 @@ if ($logfile){ > print $loghandle "$i MARC records done in $timeneeded seconds\n"; > $loghandle->close; > } >+if ($yamlfile) { >+ open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; >+ print YAML Dump($yamlhash); >+} > exit 0; > > sub GetRecordId{ >@@ -460,9 +534,21 @@ sub report_item_errors { > } > sub printlog{ > my $logelements=shift; >- print $loghandle join (";",@$logelements{qw<id op status>}),"\n"; >+ print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; >+} >+sub get_heading_fields{ >+ my $headingfields; >+ if ($authtypes){ >+ $headingfields=YAML::LoadFile($authtypes); >+ $headingfields={C4::Context->preference('marcflavour')=>$headingfields}; >+ $debug && warn YAML::Dump($headingfields); >+ } >+ unless ($headingfields){ >+ $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); >+ $headingfields={C4::Context->preference('marcflavour')=>$headingfields}; >+ } >+ return $headingfields; > } >- > > =head1 NAME > >@@ -524,7 +610,7 @@ The I<NUMBER> of records to wait before performing a 'commit' operation > > File logs actions done for each record and their status into file > >-=item B<-t> >+=item B<-t, -test> > > Test mode: parses the file, saying what he would do, but doing nothing. > >@@ -547,6 +633,30 @@ biblioitems, items > > Input file I<FORMAT>: I<MARCXML> or I<ISO2709> (defaults to ISO2709) > >+=item B<-authtypes> >+ >+file yamlfile with authoritiesTypes and distinguishable record field in order >+to store the correct authtype >+ >+=item B<-yaml> >+ >+yaml file format a yaml file with ids >+keepids field store ids in field (usefull for authorities, >+where 001 contains the authid for Koha, that can contain a very valuable >+info for authorities coming from LOC or BNF. useless for biblios probably >+ >+=item B<-insert> >+ >+if set, only insert when possible >+ >+=item B<-update> >+ >+if set, only updates (any biblio should have a matching record) >+ >+=item B<-all> >+ >+if set, do whatever is required >+ > =item B<-k, -keepids>=<FIELD> > > Field store ids in I<FIELD> (usefull for authorities, where 001 contains the >-- >1.7.9.5 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 5635
:
3547
|
7485
|
7486
|
11071
|
13671
|
14499
|
14500
|
14502
|
14503
|
14504
|
16227
|
16259
|
16364