From a2f71b30941f19834b98b5e4e7665d599552b431 Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Tue, 1 Oct 2013 14:35:23 +0300 Subject: [PATCH] Bug 6113 [ENH] - enhancement to keep previous ids Modified the C4::Biblio::_koha_add_biblioitem() && _koha_add_biblio() to support INSERTing biblionumbers and biblioitemnumbers to the DB. Also validating that id's are positive integers and well below the critical Perl LONG_MAX limit so we don't get buffer overflow. Duplicate id INSERTion will fail, but in normal use scenarios this is not an issue, if "Koha to MARC mapping" has biblionumber and biblioitemnumber mapped under field 999. MySQL takes care of the primary key sequence refreshing. Updated the bulkmarcimport.pl -script to copy the desired legacy id MARC field to the field designated in "Koha to MARC mapping" -> biblionumber && biblioitemnumber. This is handy if you have the legacy id in just 001 and need to populate 999c and 999d to push them to DB. No need to make an extra modification run to source bibliographic records just to facilitate Koha migration better. Documented the new functionality in bulkmarcimport.pl --- C4/Biblio.pm | 124 ++++++++++++++++++++++++++++----- misc/migration_tools/bulkmarcimport.pl | 77 ++++++++++++++++---- 2 files changed, 172 insertions(+), 29 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index cf83ae6..5e98f42 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -2944,9 +2944,36 @@ sub _koha_add_biblio { $biblio->{'serial'} = 0; if ( $biblio->{'seriestitle'} ) { $biblio->{'serial'} = 1 } } + + ### Validating biblionumber. + my $biblionumber; + my $isBiblioNumberOk = ''; #placeholder for the sql-clause which inserts biblionumber + if (exists $biblio->{'biblionumber'}) { + $biblionumber = $biblio->{'biblionumber'}; + if (! ($biblionumber =~ /^\d+$/) ) { + $error .= "C4::Biblio::_koha_add_biblio():> Biblionumber $biblionumber is not a number! Using a new biblionumber\n"; + warn $error; + } + elsif (! ($biblionumber > 0)) { + $error .= "C4::Biblio::_koha_add_biblio():> Biblionumber $biblionumber is not greater than 0! Using a new biblionumber\n"; + warn $error; + } + #The real Perl LONG_MAX is 2147483646, but using 2147483640 + # so we can detect the imminent primary key exhaustion before it actually is exhausted. + elsif (! ($biblionumber < 2147483640)) { + $error .= "C4::Biblio::_koha_add_biblio():> Biblionumber $biblionumber is too large for Perl! Using a new biblionumber\n"; + warn $error; + } + else { + #We have validated the biblionumber and can prepare to insert it. + $isBiblioNumberOk = 'biblionumber = ?,'; + } + + } my $query = "INSERT INTO biblio SET frameworkcode = ?, + $isBiblioNumberOk author = ?, title = ?, unititle =?, @@ -2958,16 +2985,31 @@ sub _koha_add_biblio { abstract = ? "; my $sth = $dbh->prepare($query); - $sth->execute( - $frameworkcode, $biblio->{'author'}, $biblio->{'title'}, $biblio->{'unititle'}, $biblio->{'notes'}, - $biblio->{'serial'}, $biblio->{'seriestitle'}, $biblio->{'copyrightdate'}, $biblio->{'abstract'} - ); + + if ($isBiblioNumberOk) { + $sth->execute( + $frameworkcode, $biblionumber, $biblio->{'author'}, $biblio->{'title'}, $biblio->{'unititle'}, $biblio->{'notes'}, + $biblio->{'serial'}, $biblio->{'seriestitle'}, $biblio->{'copyrightdate'}, $biblio->{'abstract'} + ); + } + else { + $sth->execute( + $frameworkcode, $biblio->{'author'}, $biblio->{'title'}, $biblio->{'unititle'}, $biblio->{'notes'}, + $biblio->{'serial'}, $biblio->{'seriestitle'}, $biblio->{'copyrightdate'}, $biblio->{'abstract'} + ); + } + - my $biblionumber = $dbh->{'mysql_insertid'}; + $biblionumber = $isBiblioNumberOk ? $biblionumber : $dbh->{'mysql_insertid'}; if ( $dbh->errstr ) { $error .= "ERROR in _koha_add_biblio $query" . $dbh->errstr; + } + + + if (defined $error) { warn $error; } + $sth->finish(); @@ -3093,9 +3135,36 @@ Internal function to add a biblioitem sub _koha_add_biblioitem { my ( $dbh, $biblioitem ) = @_; my $error; + + ### Validating biblionumber. + my $biblioitemnumber; + my $isBiblioitemNumberOk = ''; #placeholder for the sql-clause which inserts biblionumber + if (exists $biblioitem->{'biblioitemnumber'}) { + $biblioitemnumber = $biblioitem->{'biblioitemnumber'}; + if (! ($biblioitemnumber =~ /^\d+$/) ) { + $error .= "C4::Biblio::_koha_add_biblio():> biblioitemnumber $biblioitemnumber is not a number! Using a new biblioitemnumber\n"; + warn $error; + } + elsif (! ($biblioitemnumber > 0)) { + $error .= "C4::Biblio::_koha_add_biblio():> biblioitemnumber $biblioitemnumber is not greater than 0! Using a new biblioitemnumber\n"; + warn $error; + } + #The real Perl LONG_MAX is 2147483646, but using 2147483640 + # so we can detect the imminent primary key exhaustion before it actually is exhausted. + elsif (! ($biblioitemnumber < 2147483640)) { + $error .= "C4::Biblio::_koha_add_biblio():> biblioitemnumber $biblioitemnumber is too large for Perl! Using a new biblioitemnumber\n"; + warn $error; + } + else { + #We have validated the biblionumber and can prepare to insert it. + $isBiblioitemNumberOk = 'biblioitemnumber = ?,'; + } + + } my ($cn_sort) = GetClassSort( $biblioitem->{'biblioitems.cn_source'}, $biblioitem->{'cn_class'}, $biblioitem->{'cn_item'} ); my $query = "INSERT INTO biblioitems SET + $isBiblioitemNumberOk biblionumber = ?, volume = ?, number = ?, @@ -3129,24 +3198,45 @@ sub _koha_add_biblioitem { agerestriction = ? "; my $sth = $dbh->prepare($query); - $sth->execute( - $biblioitem->{'biblionumber'}, $biblioitem->{'volume'}, $biblioitem->{'number'}, $biblioitem->{'itemtype'}, - $biblioitem->{'isbn'}, $biblioitem->{'issn'}, $biblioitem->{'publicationyear'}, $biblioitem->{'publishercode'}, - $biblioitem->{'volumedate'}, $biblioitem->{'volumedesc'}, $biblioitem->{'collectiontitle'}, $biblioitem->{'collectionissn'}, - $biblioitem->{'collectionvolume'}, $biblioitem->{'editionstatement'}, $biblioitem->{'editionresponsibility'}, $biblioitem->{'illus'}, - $biblioitem->{'pages'}, $biblioitem->{'bnotes'}, $biblioitem->{'size'}, $biblioitem->{'place'}, - $biblioitem->{'lccn'}, $biblioitem->{'marc'}, $biblioitem->{'url'}, $biblioitem->{'biblioitems.cn_source'}, - $biblioitem->{'cn_class'}, $biblioitem->{'cn_item'}, $biblioitem->{'cn_suffix'}, $cn_sort, - $biblioitem->{'totalissues'}, $biblioitem->{'ean'}, $biblioitem->{'agerestriction'} - ); - my $bibitemnum = $dbh->{'mysql_insertid'}; + + if ($isBiblioitemNumberOk) { + $sth->execute( + $biblioitem->{'biblioitemnumber'}, + $biblioitem->{'biblionumber'}, $biblioitem->{'volume'}, $biblioitem->{'number'}, $biblioitem->{'itemtype'}, + $biblioitem->{'isbn'}, $biblioitem->{'issn'}, $biblioitem->{'publicationyear'}, $biblioitem->{'publishercode'}, + $biblioitem->{'volumedate'}, $biblioitem->{'volumedesc'}, $biblioitem->{'collectiontitle'}, $biblioitem->{'collectionissn'}, + $biblioitem->{'collectionvolume'}, $biblioitem->{'editionstatement'}, $biblioitem->{'editionresponsibility'}, $biblioitem->{'illus'}, + $biblioitem->{'pages'}, $biblioitem->{'bnotes'}, $biblioitem->{'size'}, $biblioitem->{'place'}, + $biblioitem->{'lccn'}, $biblioitem->{'marc'}, $biblioitem->{'url'}, $biblioitem->{'biblioitems.cn_source'}, + $biblioitem->{'cn_class'}, $biblioitem->{'cn_item'}, $biblioitem->{'cn_suffix'}, $cn_sort, + $biblioitem->{'totalissues'}, $biblioitem->{'ean'}, $biblioitem->{'agerestriction'} + ); + } + else { + $sth->execute( + $biblioitem->{'biblionumber'}, $biblioitem->{'volume'}, $biblioitem->{'number'}, $biblioitem->{'itemtype'}, + $biblioitem->{'isbn'}, $biblioitem->{'issn'}, $biblioitem->{'publicationyear'}, $biblioitem->{'publishercode'}, + $biblioitem->{'volumedate'}, $biblioitem->{'volumedesc'}, $biblioitem->{'collectiontitle'}, $biblioitem->{'collectionissn'}, + $biblioitem->{'collectionvolume'}, $biblioitem->{'editionstatement'}, $biblioitem->{'editionresponsibility'}, $biblioitem->{'illus'}, + $biblioitem->{'pages'}, $biblioitem->{'bnotes'}, $biblioitem->{'size'}, $biblioitem->{'place'}, + $biblioitem->{'lccn'}, $biblioitem->{'marc'}, $biblioitem->{'url'}, $biblioitem->{'biblioitems.cn_source'}, + $biblioitem->{'cn_class'}, $biblioitem->{'cn_item'}, $biblioitem->{'cn_suffix'}, $cn_sort, + $biblioitem->{'totalissues'}, $biblioitem->{'ean'}, $biblioitem->{'agerestriction'} + ); + } + + $biblioitemnumber = $isBiblioitemNumberOk ? $biblioitemnumber : $dbh->{'mysql_insertid'}; if ( $dbh->errstr ) { $error .= "ERROR in _koha_add_biblioitem $query" . $dbh->errstr; + } + + if (defined $error) { warn $error; } + $sth->finish(); - return ( $bibitemnum, $error ); + return ( $biblioitemnumber, $error ); } =head2 _koha_delete_biblio diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index c0cc0e4..450b91e 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -34,7 +34,7 @@ use Pod::Usage; use open qw( :std :encoding(UTF-8) ); binmode( STDOUT, ":encoding(UTF-8)" ); my ( $input_marc_file, $number, $offset) = ('',0,0); -my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); +my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile, $legacyIdAsBiblionumberFromHere); my ( $insert, $filters, $update, $all, $yamlfile, $authtypes ); my $cleanisbn = 1; my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); @@ -73,6 +73,7 @@ GetOptions( 'yaml:s' => \$yamlfile, 'dedupbarcode' => \$dedup_barcode, 'framework=s' => \$framework, + 'g|legacyIdAsBiblionumberFromHere:s' => \$legacyIdAsBiblionumberFromHere, ); $biblios ||= !$authorities; $insert ||= !$update; @@ -166,14 +167,23 @@ if ( $offset ) { $batch->next() while ($offset--); } -my ($tagid,$subfieldid); +my ($tagid, $subfieldid, $binum_tagid, $binum_subfieldid); if ($authorities){ $tagid='001'; } else { ( $tagid, $subfieldid ) = GetMarcFromKohaField( "biblio.biblionumber", $framework ); + ( $binum_tagid, $binum_subfieldid ) = + GetMarcFromKohaField( "biblioitems.biblioitemnumber", $framework ); $tagid||="001"; + + #This is important for the functionality $legacyIdAsBiblionumberFromHere + if ($tagid ne $binum_tagid) { + warn "Koha to MARC mapping values biblio.biblionumber and biblioitems.biblioitemnumber must share the same MARC field!"; + exit(1); + } + } # the SQL query to search on isbn @@ -207,16 +217,23 @@ RECORD: while ( ) { - ### Open library hack! yay ### - ### Move field 001 to 999c and 999d ### - my $f001 = $record->field('001'); - $f001 = $f001->data(); - my $f999 = MARC::Field->new( 999, '', '', - 'c' => $f001, - 'd' => $f001 - ); - $record->append_fields($f999); - + ### If you are like me and don't have the legacy id in $999c and $999d, you + ### can use this functionality to copy the id on the fly. + ### By default, copy legacy id field 001 to 999c and 999d ### + if ($legacyIdAsBiblionumberFromHere) { + my $legacyIdField = $record->field( $legacyIdAsBiblionumberFromHere ); + if ($legacyIdField) { + my $legacyId = $legacyIdField->data(); + my $biblionumberField = MARC::Field->new( $tagid, '', '', + $subfieldid => $legacyId, + $binum_subfieldid => $legacyId, + ); + $record->append_fields($biblionumberField); + } + else { + warn "No $legacyIdAsBiblionumberFromHere found!"; + } + } # transcode the record to UTF8 if needed & applicable. if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { @@ -592,6 +609,14 @@ bulkmarcimport.pl - Import bibliographic/authority records into Koha $ export KOHA_CONF=/etc/koha.conf $ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\ -file /home/jmf/koha.mrc -n 3000 + + Migrate your legacy bibliographic records to Koha, while having the legacy id + in a field $999 according to your directives here + "Administration" > "Koha to Marc mapping". + You also have the legacy id in field $001 so your analytic and component + records links wont be broken. + $ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 \\ + -file /home/jmf/koha.marcxml -m MARCXML -b =head1 WARNING @@ -599,6 +624,15 @@ Don't use this script before you've entered and checked your MARC parameters tables twice (or more!). Otherwise, the import won't work correctly and you will get invalid data. +If you want to assign your legacy bibliographic id as the biblionumber and +biblioitemnumber in Koha biblio- and bibliotiems -tables, you must first +specify the mappings in "Administration" > "Koha to MARC mapping". + +It is extremely recommended to use field $999 to migrate your legacy id's. +If you set it to for ex. 001, librarians might manually reuse an existing id. +Also make sure that the legacy id is a strict positive integer and less than +perl -MPOSIX -le 'print LONG_MAX'. + =head1 DESCRIPTION =over @@ -698,6 +732,12 @@ Field store ids in I (usefull for authorities, where 001 contains the authid for Koha, that can contain a very valuable info for authorities coming from LOC or BNF. useless for biblios probably) +If your legacy id is already in 001, or the field you want it in. Dont use this +parameter as it will move the biblionumber from your definition at +"Administration" > "Koha to Marc mapping" as a duplicate of your original id. +Thus you lose your MARC-field mapped to biblionumber before the record is +processed. + =item B<-match>= I matchindex,fieldtomatch matchpoint to use to deduplicate fieldtomatch @@ -745,6 +785,19 @@ This is the code for the framework that the requested records will have attached to them when they are created. If not specified, then the default framework will be used. +=item B<-g, -legacyIdAsBiblionumberFromHere>= + +If you already have your legacy database id in your marc record and want to +preserve it, but still want to get it set as the biblionumber and biblioitemnumber +in Kohas database, use this. + +Example: +Field 001 has your legacy id. +Koha to MARC mapping needs legacy id in 999c and 999d as well for biblionumber +and biblioitemnumber, use + -g 001 +to copy the legacy id to field 999cd and thus get the id preserved in DB. + =back =cut -- 1.8.1.2