From a771112c8f776b150481ec4e8f0b4ac0f525a5be Mon Sep 17 00:00:00 2001 From: Henri-Damien LAURENT Date: Tue, 7 Feb 2012 17:24:04 +0100 Subject: [PATCH] Bug 8304 : composed authorities Rameau management Content-Type: text/plain; charset="UTF-8" Bug 5635: bulkmarcimport new parameters & features This patch is designed to manage the composed authorities the Way Rameau is coping with them in France. This enables libraries to merge composed authorities and only change the heading field. It supposes that the previous data is sent to merge function so that process deletes previous heading entries. Say you have in a biblio record : - 606 $9 1214 $3 123 $a Europe $9 1215 $3 456 $y France $9 1210 $3 199 $x Politiciens $9 1216 $3 789 $z 1801-1860 And you change authority 1210 -250 $a Politiciens into -250 $a Politique You should get : - 606 $9 1214 $3 123 $a Europe $9 1215 $3 456 $y France $9 1210 $3 199 $x Politique $9 1216 $3 789 $z 1801-1860 In an other record : - 606 $9 1210 $3 199 $a Politiciens You should get : - 606 $9 1210 $3 199 $a Politique This patch changes the merge function. And adds three internal functions. merge function has been prettyfied And documentation from GuessAuthTypeCode has been updated (had not been) --- C4/AuthoritiesMarc.pm | 303 +++++++++++++++++++++----------- misc/migration_tools/bulkmarcimport.pl | 217 +++++++++++++++++------ 2 files changed, 368 insertions(+), 152 deletions(-) diff --git a/C4/AuthoritiesMarc.pm b/C4/AuthoritiesMarc.pm index 3b08d80..438e048 100644 --- a/C4/AuthoritiesMarc.pm +++ b/C4/AuthoritiesMarc.pm @@ -19,12 +19,14 @@ package C4::AuthoritiesMarc; use strict; use warnings; use C4::Context; +use List::MoreUtils qw/any none/; use MARC::Record; use C4::Biblio; use C4::Search; use C4::AuthoritiesMarc::MARC21; use C4::AuthoritiesMarc::UNIMARC; use C4::Charset; +use C4::Debug; use C4::Log; use vars qw($VERSION @ISA @EXPORT); @@ -332,6 +334,10 @@ sub SearchAuthorities { my $authrecord; my $separator=C4::Context->preference('authoritysep'); $authrecord = MARC::File::USMARC::decode($marcdata); + unless ($authrecord->field('001')) { + $nbresults--; + next; + } my $authid=$authrecord->field('001')->data(); my %newline; $newline{authid} = $authid; @@ -440,16 +446,16 @@ sub GetAuthTypeCode { =head2 GuessAuthTypeCode - my $authtypecode = GuessAuthTypeCode($record); + my $authtypecode = GuessAuthTypeCode($record, [$headingfields]); Get the record and tries to guess the adequate authtypecode from its content. =cut sub GuessAuthTypeCode { - my ($record) = @_; + my ($record, $heading_fields) = @_; return unless defined $record; -my $heading_fields = { + $heading_fields //= { "MARC21"=>{ '100'=>{authtypecode=>'PERSO_NAME'}, '110'=>{authtypecode=>'CORPO_NAME'}, @@ -1289,6 +1295,7 @@ sub AddAuthorityTrees{ return $rq->execute($trees,$authid); } + =head2 merge $ref= &merge(mergefrom,$MARCfrom,$mergeto,$MARCto) @@ -1298,10 +1305,58 @@ Then we should add some new parameter : bibliotargettag, authtargettag =cut +# Test if a subfield exist in a subfield list +sub _test_subfcode_presence { + my ( $subfields, $subfcode ) = @_; + return grep { $_->[0] eq $subfcode } @$subfields; +} + +# Test if a string exists in a field +sub _test_string { + my ( $string, @fields ) = @_; + my $return = 0; + for my $field (@fields) { + if ( grep { $_->[1] =~ /$string/i } $field->subfields ) { + $return = 1; + last; + } + } + return $return; +} + +# _process_subfcode_4_merge sets the correct subfcode +sub _process_subfcode_4_merge { + my ( $tagfield, $bibliosubfields, $authorityrecord, $authoritysubfields, + $authtypecode ) + = @_; + return unless ( uc( C4::Context->preference('marcflavour') ) eq 'UNIMARC' ); + my $authtag = GetAuthType($authtypecode); + my $chronological_auth = + _test_string( qq{S'emploie uniquement en subdivision chronologique}, + $authorityrecord->field('3..') ); + my $subfz_absent = not _test_subfcode_presence( $bibliosubfields, 'z' ); + if ( _test_subfcode_presence( $bibliosubfields, "a" ) ) { + if ( ( $authtag->{'auth_tag_to_report'} eq '215' ) + and ( $tagfield ne "607" ) ) + { + return "y"; + + } + elsif ( $chronological_auth and $subfz_absent ) { + return "z"; + } + else { + return "x"; + } + } + return; +} + sub merge { my ($mergefrom,$MARCfrom,$mergeto,$MARCto) = @_; my ($counteditedbiblio,$countunmodifiedbiblio,$counterrors)=(0,0,0); my $dbh=C4::Context->dbh; + my $marcflavour = C4::Context->preference('marcflavour'); my $authtypecodefrom = GetAuthTypeCode($mergefrom); my $authtypecodeto = GetAuthTypeCode($mergeto); # warn "mergefrom : $authtypecodefrom $mergefrom mergeto : $authtypecodeto $mergeto "; @@ -1316,11 +1371,22 @@ sub merge { my ($auth_tag_to_report_to) = $sth->fetchrow; my @record_to; - @record_to = $MARCto->field($auth_tag_to_report_to)->subfields() if $MARCto->field($auth_tag_to_report_to); + @record_to = + grep { $_->[0] !~ /[0-9]/ } + $MARCto->field($auth_tag_to_report_to)->subfields() + if $MARCto->field($auth_tag_to_report_to); + if ( uc($marcflavour) eq 'UNIMARC' and $MARCto->field('009') ) { + unshift @record_to, [ 3 => $MARCto->field('009')->data() ]; + } + $debug and warn 'fields to merge', Data::Dumper::Dumper(@record_to); + my $field_to; + $field_to = $MARCto->field($auth_tag_to_report_to) + if $MARCto->field($auth_tag_to_report_to); my @record_from; @record_from = $MARCfrom->field($auth_tag_to_report_from)->subfields() if $MARCfrom->field($auth_tag_to_report_from); my @reccache; + # search all biblio tags using this authority. #Getting marcbiblios impacted by the change. if (C4::Context->preference('NoZebra')) { @@ -1356,58 +1422,135 @@ sub merge { my $marcdata = $rec->raw(); my $marcrecordzebra= MARC::Record->new_from_usmarc($marcdata); my ( $biblionumbertagfield, $biblionumbertagsubfield ) = &GetMarcFromKohaField( "biblio.biblionumber", '' ); - my $i = ($biblionumbertagfield < 10) ? $marcrecordzebra->field($biblionumbertagfield)->data : $marcrecordzebra->subfield($biblionumbertagfield, $biblionumbertagsubfield); - my $marcrecorddb=GetMarcBiblio($i); + my $bibnum; + if ($biblionumbertagfield < 10) { + $bibnum = $marcrecordzebra->field($biblionumbertagfield)->data; + } + else { + $bibnum = $marcrecordzebra->subfield($biblionumbertagfield, $biblionumbertagsubfield); + } + my $marcrecorddb=GetMarcBiblio($bibnum); push @reccache, $marcrecorddb; $z++; } $oResult->destroy(); } + #warn scalar(@reccache)." biblios to update"; # Get All candidate Tags for the change # (This will reduce the search scope in marc records). - $sth = $dbh->prepare("select distinct tagfield from marc_subfield_structure where authtypecode=?"); - $sth->execute($authtypecodefrom); + $sth = $dbh->prepare( +"select distinct tagfield from marc_subfield_structure where authtypecode<>''" + ); + $sth->execute(); my @tags_using_authtype; while (my ($tagfield) = $sth->fetchrow) { push @tags_using_authtype,$tagfield ; } - my $tag_to=0; - if ($authtypecodeto ne $authtypecodefrom){ - # If many tags, take the first - $sth->execute($authtypecodeto); - $tag_to=$sth->fetchrow; - #warn $tag_to; - } - # BulkEdit marc records - # May be used as a template for a bulkedit field - foreach my $marcrecord(@reccache){ - my $update; - foreach my $tagfield (@tags_using_authtype){ -# warn "tagfield : $tagfield "; - foreach my $field ($marcrecord->field($tagfield)){ - my $auth_number=$field->subfield("9"); - my $tag=$field->tag(); - if ($auth_number==$mergefrom) { - my $field_to=MARC::Field->new(($tag_to?$tag_to:$tag),$field->indicator(1),$field->indicator(2),"9"=>$mergeto); - my $exclude='9'; - foreach my $subfield (@record_to) { - $field_to->add_subfields($subfield->[0] =>$subfield->[1]); - $exclude.= $subfield->[0]; + my $tag_to = 0; + + # if ((defined $authtypecodeto) and ($authtypecodeto ne $authtypecodefrom)){ + # # If many tags, take the first + # $sth->execute($authtypecodeto); + # $tag_to=$sth->fetchrow; + # #warn $tag_to; + # } + foreach my $marcrecord (@reccache) { + foreach my $tagfield (@tags_using_authtype) { + foreach my $field ( $marcrecord->field($tagfield) ) { + my $update; + my $tag = $field->tag(); + my @newsubfields; + my %indexes; + + #get to next field if no subfield + my @localsubfields = $field->subfields(); + my $index_9_auth = 0; + my $found = 0; + for my $subf (@localsubfields) { + + # $debug && warn Data::Dumper::Dumper($subf); + if ( ( $subf->[0] eq "9" ) + and ( $subf->[1] == $mergefrom ) ) + { + $found = 1; + $debug && warn "found $mergefrom " . $subf->[1]; + last; + } + $index_9_auth++; } - $exclude='['.$exclude.']'; -# add subfields in $field not included in @record_to - my @restore= grep {$_->[0]!~/$exclude/} $field->subfields(); - foreach my $subfield (@restore) { - $field_to->add_subfields($subfield->[0] =>$subfield->[1]); - } - $marcrecord->delete_field($field); - $marcrecord->insert_grouped_field($field_to); - $update=1; + + #$debug && warn "$index_9_auth $#localsubfields $found"; + next if ( $index_9_auth >= $#localsubfields and !$found ); + + # Removes the data if before the $9 + my $index = 0; + for my $subf ( @localsubfields[ 0 .. $index_9_auth ] ) { + if ( any { $subf->[1] eq $_->[1] } @record_from ) { + $debug && warn "found $subf->[0] " . $subf->[1]; + splice @localsubfields, $index, 1; + $index_9_auth--; + } + else { + $index++; + } + } + + #Get the next $9 subfield + my $nextindex_9 = 0; + for my $subf ( + @localsubfields[ $index_9_auth + 1 .. $#localsubfields ] ) + { + last if ( $subf->[0] =~ /[12456789]/ ); + $nextindex_9++; + } + + #Change the first tag if required + # That is : change the first tag ($a) to what it is in the biblio record + # Since some composed authorities will place the $a into $x or $y + + my @previous_subfields = @localsubfields[ 0 .. $index_9_auth ]; + if ( + my $changesubfcode = _process_subfcode_4_merge( + $tag, \@previous_subfields, + $MARCto, \@record_to, + $authtypecodeto + ) + ) + { + $record_to[0]->[0] = $changesubfcode + if defined($changesubfcode); } - }#for each tag - }#foreach tagfield - my ($bibliotag,$bibliosubf) = GetMarcFromKohaField("biblio.biblionumber","") ; + +#$debug && warn "$index_9_auth $nextindex_9 data to add ".Data::Dumper::Dumper(@record_to); +# Replace in local subfields the subfields related to recordfrom with data from record_to + + my @localrecord_to = @record_to; + @localrecord_to = ( [ 9, $mergeto ], @localrecord_to ); + + #$debug && warn "localrecordto ".Data::Dumper::Dumper(@localrecord_to); + splice( + @localsubfields, $index_9_auth, + $nextindex_9 + 1, @localrecord_to + ); + + #$debug && warn "after splice ".Data::Dumper::Dumper(@localsubfields); + #very nice api for MARC::Record + # It seems that some elements localsubfields can be undefined so skip them + @newsubfields = + map { ( defined $_ ? @$_ : () ) } @localsubfields; + + #filter to subfields which are not in the subfield + my $field_to = MARC::Field->new( + ( $tag_to ? $tag_to : $tag ), $field->indicator(1), + $field->indicator(2), @newsubfields + ); + $marcrecord->delete_field($field); + $marcrecord->insert_fields_ordered($field_to); + } #for each tag + } #foreach tagfield + my ( $bibliotag, $bibliosubf ) = + GetMarcFromKohaField( "biblio.biblionumber", "" ); my $biblionumber; if ($bibliotag<10){ $biblionumber=$marcrecord->field($bibliotag)->data; @@ -1415,66 +1558,26 @@ sub merge { else { $biblionumber=$marcrecord->subfield($bibliotag,$bibliosubf); } - unless ($biblionumber){ - warn "pas de numéro de notice bibliographique dans : ".$marcrecord->as_formatted; + + # $debug && warn $biblionumber,$marcrecord->as_formatted; + unless ($biblionumber) { + warn "no biblionumber in: " + . $marcrecord->as_formatted; next; } - if ($update==1){ - &ModBiblio($marcrecord,$biblionumber,GetFrameworkCode($biblionumber)) ; - $counteditedbiblio++; - warn $counteditedbiblio if (($counteditedbiblio % 10) and $ENV{DEBUG}); - } - }#foreach $marc - return $counteditedbiblio; - # now, find every other authority linked with this authority - # now, find every other authority linked with this authority -# my $oConnection=C4::Context->Zconn("authorityserver"); -# my $query; -# # att 9210 Auth-Internal-authtype -# # att 9220 Auth-Internal-LN -# # ccl.properties to add for authorities -# $query= "= ".$mergefrom; -# my $oResult = $oConnection->search(new ZOOM::Query::CCL2RPN( $query, $oConnection )); -# my $count=$oResult->size() if ($oResult); -# my @reccache; -# my $z=0; -# while ( $z<$count ) { -# my $rec; -# $rec=$oResult->record($z); -# my $marcdata = $rec->raw(); -# push @reccache, $marcdata; -# $z++; -# } -# $oResult->destroy(); -# foreach my $marc(@reccache){ -# my $update; -# my $marcrecord; -# $marcrecord = MARC::File::USMARC::decode($marc); -# foreach my $tagfield (@tags_using_authtype){ -# $tagfield=substr($tagfield,0,3); -# my @tags = $marcrecord->field($tagfield); -# foreach my $tag (@tags){ -# my $tagsubs=$tag->subfield("9"); -# #warn "$tagfield:$tagsubs:$mergefrom"; -# if ($tagsubs== $mergefrom) { -# $tag->update("9" =>$mergeto); -# foreach my $subfield (@record_to) { -# # warn "$subfield,$subfield->[0],$subfield->[1]"; -# $tag->update($subfield->[0] =>$subfield->[1]); -# }#for $subfield -# } -# $marcrecord->delete_field($tag); -# $marcrecord->add_fields($tag); -# $update=1; -# }#for each tag -# }#foreach tagfield -# my $authoritynumber = TransformMarcToKoha($dbh,$marcrecord,"") ; -# if ($update==1){ -# &ModAuthority($marcrecord,$authoritynumber,GetAuthTypeCode($authoritynumber)) ; -# } -# -# }#foreach $marc -}#sub + + #if ( $update == 1 ) { + &ModBiblio( $marcrecord, $biblionumber, + GetFrameworkCode($biblionumber) ); + $counteditedbiblio++; + warn $counteditedbiblio + if ( ( $counteditedbiblio % 10 ) and $ENV{DEBUG} ); + + #} + } #foreach $marc + DelAuthority($mergefrom) if ( $mergefrom != $mergeto ); + return $counteditedbiblio; +} #sub =head2 get_auth_type_location diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index a9fee95..c1b5cfe 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -24,15 +24,19 @@ use C4::Koha; use C4::Debug; use C4::Charset; use C4::Items; +use YAML; use Unicode::Normalize; use Time::HiRes qw(gettimeofday); use Getopt::Long; use IO::File; use Pod::Usage; -binmode STDOUT, ':encoding(UTF-8)'; +use open qw( :std :utf8 ); +binmode( STDOUT, ":utf8" ); my ( $input_marc_file, $number, $offset) = ('',0,0); my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); + +my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); my $cleanisbn = 1; my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); @@ -45,7 +49,7 @@ GetOptions( 'o|offset:f' => \$offset, 'h' => \$version, 'd' => \$delete, - 't' => \$test_parameter, + 't|test' => \$test_parameter, 's' => \$skip_marc8_conversion, 'c:s' => \$char_encoding, 'v:s' => \$verbose, @@ -55,6 +59,11 @@ GetOptions( 'k|keepids:s' => \$keepids, 'b|biblios' => \$biblios, 'a|authorities' => \$authorities, + 'authtypes:s' => \$authtypes, + 'filter=s@' => \$filters, + 'insert' => \$insert, + 'update' => \$update, + 'all' => \$all, 'match=s@' => \$match, 'i|isbn' => \$isbn_check, 'x:s' => \$sourcetag, @@ -62,14 +71,23 @@ GetOptions( 'idmap:s' => \$idmapfl, 'cleanisbn!' => \$cleanisbn, 'dedupbarcode' => \$dedup_barcode, + 'yaml:s' => \$yamlfile, ); -$biblios=!$authorities||$biblios; +$biblios ||= !$authorities; +$insert ||= !$update; + +if ($all) { + $insert = 1; + $update = 1; +} if ($version || ($input_marc_file eq '')) { pod2usage( -verbose => 2 ); exit; } +my $heading_fields=get_heading_fields(); + if (defined $idmapfl) { open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n"; } @@ -140,7 +158,7 @@ $batch->warnings_off(); $batch->strict_off(); my $i=0; my $commitnum = $commit ? $commit : 50; - +my $yamlhash; # Skip file offset if ( $offset ) { @@ -197,6 +215,7 @@ RECORD: while ( ) { next RECORD; } } + SetUTF8Flag($record); my $isbn; # remove trailing - in isbn (only for biblios, of course) if ($biblios && $cleanisbn) { @@ -210,45 +229,76 @@ RECORD: while ( ) { } my $id; # search for duplicates (based on Local-number) - if ($match){ - require C4::Search; - my $query=build_query($match,$record); - my $server=($authorities?'authorityserver':'biblioserver'); - my ($error, $results,$totalhits)=C4::Search::SimpleSearch( $query, 0, 3, [$server] ); - die "unable to search the database for duplicates : $error" if (defined $error); - #warn "$query $server : $totalhits"; - if ( @{$results} == 1 ){ - my $marcrecord = MARC::File::USMARC::decode($results->[0]); - $id=GetRecordId($marcrecord,$tagid,$subfieldid); - } - elsif ( @{$results} > 1){ - $debug && warn "more than one match for $query"; - } - else { - $debug && warn "nomatch for $query"; - } + my $originalid; + $originalid = GetRecordId( $record, $tagid, $subfieldid ); + if ($match) { + require C4::Search; + my $query = build_query( $match, $record ); + my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); + $debug && warn $query; + my ( $error, $results, $totalhits ) = C4::Search::SimpleSearch( $query, 0, 3, [$server] ); + die "unable to search the database for duplicates : $error" if ( defined $error ); + $debug && warn "$query $server : $totalhits"; + if ( $results && scalar(@$results) == 1 ) { + my $marcrecord = MARC::File::USMARC::decode( $results->[0] ); + SetUTF8Flag($marcrecord); + $id = GetRecordId( $marcrecord, $tagid, $subfieldid ); + if ( $authorities && $marcFlavour ) { + #Skip if authority in database is the same as the on in database + if ( $marcrecord->field('005')->data >= $record->field('005')->data ) { + if ($yamlfile) { + $yamlhash->{$originalid}->{'authid'} = $id; + + # On récupère tous les souschamps des champs vedettes d'autorités + my @subfields; + foreach my $field ( $marcrecord->field("2..") ) { + push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); + } + $yamlhash->{$originalid}->{'subfields'} = \@subfields; + } + next; + } + } + } elsif ( $results && scalar(@$results) > 1 ) { + $debug && warn "more than one match for $query"; + } else { + $debug && warn "nomatch for $query"; + } } - my $originalid; - if ($keepids){ - $originalid=GetRecordId($record,$tagid,$subfieldid); - if ($originalid){ - my $storeidfield; - if (length($keepids)==3){ - $storeidfield=MARC::Field->new($keepids,$originalid); - } - else { - $storeidfield=MARC::Field->new(substr($keepids,0,3),"","",substr($keepids,3,1),$originalid); - } - $record->insert_fields_ordered($storeidfield); - $record->delete_field($record->field($tagid)); - } + if ($keepids && $originalid) { + my $storeidfield; + if ( length($keepids) == 3 ) { + $storeidfield = MARC::Field->new( $keepids, $originalid ); + } else { + $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); + } + $record->insert_fields_ordered($storeidfield); + $record->delete_field( $record->field($tagid) ); } + foreach my $stringfilter (@$filters) { + if ( length($stringfilter) == 3 ) { + foreach my $field ( $record->field($stringfilter) ) { + $record->delete_field($field); + $debug && warn "removed : ", $field->as_string; + } + } else { + my ( $removetag, $removesubfield, $removematch ) = ( $1, $2, $3 ) + if $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/; + if ( ( $removetag > "010" ) && $removesubfield ) { + foreach my $field ( $record->field($removetag) ) { + $field->delete_subfield( code => "$removesubfield", match => $removematch ); + $debug && warn "Potentially removed : ", $field->subfield($removesubfield); + } + } + } + } + unless ($test_parameter) { if ($authorities){ use C4::AuthoritiesMarc; - my $authtypecode=GuessAuthTypeCode($record); + my $authtypecode=GuessAuthTypeCode($record, $heading_fields); my $authid= ($id?$id:GuessAuthId($record)); - if ($authid && GetAuthority($authid)){ + if ($authid && GetAuthority($authid) && $update ){ ## Authority has an id and is in database : Replace eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; if ($@){ @@ -281,6 +331,14 @@ RECORD: while ( ) { printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); } } + if ($yamlfile) { + $yamlhash->{$originalid}->{'authid'} = $authid; + my @subfields; + foreach my $field ( $record->field("2..") ) { + push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); + } + $yamlhash->{$originalid}->{'subfields'} = \@subfields; + } } else { my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); @@ -304,20 +362,33 @@ RECORD: while ( ) { } # create biblio, unless we already have it ( either match or isbn ) if ($biblionumber) { - eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};} - } - else - { - eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio($record, '', { defer_marc_save => 1 }) }; + eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};}; + if ($update) { + eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkcode($biblionumber) ) }; + if ($@) { + warn "ERROR: Edit biblio $biblionumber failed: $@\n"; + printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); + next RECORD; + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); + } + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); + } + } else { + if ($insert) { + eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) }; + if ($@) { + warn "ERROR: Adding biblio $biblionumber failed: $@\n"; + printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); + next RECORD; + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); + } + } else { + printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); + } } - if ( $@ ) { - warn "ERROR: Adding biblio $biblionumber failed: $@\n"; - printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ERROR"}) if ($logfile); - next RECORD; - } - else{ - printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile); - } eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; my $error_adding = $@; # Work on a clone so that if there are real errors, we can maybe @@ -382,6 +453,7 @@ RECORD: while ( ) { if ($#{ $errors_ref } > -1) { report_item_errors($biblionumber, $errors_ref); } + $yamlhash->{$originalid} = $biblionumber if ($yamlfile); } $dbh->commit() if (0 == $i % $commitnum); } @@ -405,6 +477,10 @@ if ($logfile){ print $loghandle "$i MARC records done in $timeneeded seconds\n"; $loghandle->close; } +if ($yamlfile) { + open YAML, "> $yamlfile" or die "cannot open $yamlfile \n"; + print YAML Dump($yamlhash); +} exit 0; sub GetRecordId{ @@ -460,9 +536,22 @@ sub report_item_errors { } sub printlog{ my $logelements=shift; - print $loghandle join (";",@$logelements{qw}),"\n"; + print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw} ), "\n"; } +sub get_heading_fields{ + my $headingfields; + if ($authtypes){ + $headingfields=YAML::LoadFile($authtypes); + $headingfields={C4::Context->preference('marcflavour')=>$headingfields}; + $debug && warn YAML::Dump($headingfields); + } + unless ($headingfields){ + $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); + $headingfields={C4::Context->preference('marcflavour')=>$headingfields}; + } + return $headingfields; +} =head1 NAME @@ -524,7 +613,7 @@ The I of records to wait before performing a 'commit' operation File logs actions done for each record and their status into file -=item B<-t> +=item B<-t, -test> Test mode: parses the file, saying what he would do, but doing nothing. @@ -547,6 +636,30 @@ biblioitems, items Input file I: I or I (defaults to ISO2709) +=item B<-authtypes> + +file yamlfile with authoritiesTypes and distinguishable record field in order +to store the correct authtype + +=item B<-yaml> + +yaml file format a yaml file with ids +keepids field store ids in field (usefull for authorities, +where 001 contains the authid for Koha, that can contain a very valuable +info for authorities coming from LOC or BNF. useless for biblios probably + +=item B<-insert> + +if set, only insert when possible + +=item B<-update> + +if set, only updates (any biblio should have a matching record) + +=item B<-all> + +if set, do whatever is required + =item B<-k, -keepids>= Field store ids in I (usefull for authorities, where 001 contains the -- 1.7.2.5