From 7374fdc42b2dc743ed28ac94a192fee57406dd9f Mon Sep 17 00:00:00 2001 From: Kyle M Hall Date: Wed, 19 Jun 2013 12:25:02 -0400 Subject: [PATCH] WIP - Improve ISBN Matching http://bugs.koha-community.org/show_bug.cgi?id=10500 --- C4/Biblio.pm | 131 ++++++++++++++++++++++++++++++++ C4/ImportBatch.pm | 3 +- C4/Matcher.pm | 8 ++- installer/data/mysql/kohastructure.sql | 10 +- installer/data/mysql/updatedatabase.pl | 12 +++ t/db_dependent/Biblio.t | 18 +++-- 6 files changed, 167 insertions(+), 15 deletions(-) diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 7af8619..2017fac 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -29,6 +29,7 @@ use MARC::File::USMARC; use MARC::File::XML; use POSIX qw(strftime); use Module::Load::Conditional qw(can_load); +use Business::ISBN; use C4::Koha; use C4::Dates qw/format_date/; @@ -73,6 +74,10 @@ BEGIN { &GetMarcControlnumber &GetMarcNotes &GetMarcISBN + &GetVariationsOfISBNs + &GetVariationsOfISBN + &GetNormalizedISBNsFromMarc + &NormalizeISBN &GetMarcISSN &GetMarcSubjects &GetMarcBiblio @@ -1663,6 +1668,130 @@ sub GetMarcISBN { } # end GetMarcISBN +=head2 GetNormalizedISBNsFromMarc + + my @isbns = GetNormalizedISBNsFromMarc( $record ); + + Returns a list of normalized ISBNs from a given record + or, in a scalar context, returns a string of normalized + isbns separated by ' | ' + +=cut + +sub GetNormalizedISBNsFromMarc { + my ($record) = @_; + my ( $marc_field, $marc_subfield ) = + GetMarcFromKohaField('biblioitems.isbn'); + + my @isbns; + foreach my $subfield ( $record->field( $marc_field, $marc_subfield ) ) { + my $isbn = NormalizeISBN( { isbn => $subfield->as_string } ); + push( @isbns, $isbn ) if ($isbn); + } + + # remove duplicate isbns + @isbns = keys %{ { map { $_ => 1 } @isbns } }; + + return wantarray ? @isbns : join( " | ", @isbns ); +} + + +=head2 GetVariationsOfISBN + + my @isbns = GetVariationsOfISBN( $isbn ); + + Returns a list of varations of the given isbn in + both ISBN-10 and ISBN-13 formats, with and without + hyphens. + + In a scalar context, the isbns are returned as a + string delimited by ' | '. + +=cut + +sub GetVariationsOfISBN { + my ($isbn) = @_; + + my @isbns; + + push( @isbns, NormalizeISBN({ isbn => $isbn }) ); + push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-10' }) ); + push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-13' }) ); + push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-10', strip_hyphens => 1 }) ); + push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-13', strip_hyphens => 1 }) ); + + return wantarray ? @isbns : join( " | ", @isbns ); +} + +=head2 GetVariationsOfISBNs + + my @isbns = GetVariationsOfISBNs( @isbns ); + + Returns a list of varations of the given isbns in + both ISBN-10 and ISBN-13 formats, with and without + hyphens. + + In a scalar context, the isbns are returned as a + string delimited by ' | '. + +=cut + +sub GetVariationsOfISBNs { + my (@isbns) = @_; + + @isbns = map { GetVariationsOfISBN( $_ ) } @isbns; + + # remove duplicate isbns + @isbns = keys %{ { map { $_ => 1 } @isbns } }; + + return wantarray ? @isbns : join( " | ", @isbns ); +} + +=head2 NormalizedISBN + + my $isbns = NormalizedISBN({ + isbn => $isbn, + strip_hyphens => [0,1], + format => ['ISBN-10', 'ISBN-13'] + }); + + Returns an isbn validated by Business::ISBN. + Optionally strips hyphens and/or forces the isbn + to be of the specified format. + + If the string cannot be validated as an isbn, + it returns nothing. + +=cut + +sub NormalizeISBN { + my ($params) = @_; + + my $string = $params->{isbn}; + my $strip_hyphens = $params->{strip_hyphens}; + my $format = $params->{format}; + + my $isbn = Business::ISBN->new($string); + + if ( $isbn && $isbn->error != Business::ISBN::BAD_ISBN ) { + + if ( $format eq 'ISBN-10' ) { + $isbn = $isbn->as_isbn10(); + } + elsif ( $format eq 'ISBN-13' ) { + $isbn = $isbn->as_isbn13(); + } + + if ($strip_hyphens) { + $string = $isbn->as_string( [] ); + } else { + $string = $isbn->as_string(); + } + + return $string; + } +} + =head2 GetMarcISSN $marcissnsarray = GetMarcISSN( $record, $marcflavour ); @@ -2569,6 +2698,8 @@ sub TransformMarcToKoha { } } + $result->{'isbn'} = GetNormalizedISBNsFromMarc( $record ); + return $result; } diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 3ce5c67..5f2c2f3 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -323,7 +323,7 @@ sub ModAuthInBatch { =cut -sub BatchStageMarcRecords { +sub BatchStageMarcRecords { my $record_type = shift; my $encoding = shift; my $marc_records = shift; @@ -1399,7 +1399,6 @@ sub _add_biblio_fields { my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record); my $dbh = C4::Context->dbh; # FIXME no controlnumber, originalsource - $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)"); $sth->execute($import_record_id, $title, $author, $isbn, $issn); $sth->finish(); diff --git a/C4/Matcher.pm b/C4/Matcher.pm index 4e064be..b407a55 100644 --- a/C4/Matcher.pm +++ b/C4/Matcher.pm @@ -630,8 +630,11 @@ sub get_matches { $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser')); foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) { my @source_keys = _get_match_keys( $source_record, $matchpoint ); + next if scalar(@source_keys) == 0; + @source_keys = C4::Biblio::GetVariationsOfISBNs( @source_keys ) if ( $matchpoint->{index} eq 'isbn' ); + # build query my $query; my $error; @@ -640,13 +643,14 @@ sub get_matches { if ( $self->{'record_type'} eq 'biblio' ) { if ($QParser) { $query = join( " || ", - map { "$matchpoint->{'index'}:$_" } @source_keys ); + map { "$matchpoint->{'index'},phr:$_" } @source_keys ); } else { $query = join( " or ", - map { "$matchpoint->{'index'}=$_" } @source_keys ); + map { "$matchpoint->{'index'},phr=$_" } @source_keys ); } require C4::Search; + ( $error, $searchresults, $total_hits ) = C4::Search::SimpleSearch( $query, 0, $max_matches ); } diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index d7eebdd..1ca4373 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -152,7 +152,7 @@ CREATE TABLE `biblioitems` ( -- information related to bibliographic records in `volume` mediumtext, `number` mediumtext, `itemtype` varchar(10) default NULL, -- biblio level item type (MARC21 942$c) - `isbn` varchar(30) default NULL, -- ISBN (MARC21 020$a) + `isbn` varchar(255) default NULL, -- ISBN (MARC21 020$a) `issn` varchar(9) default NULL, -- ISSN (MARC21 022$a) `ean` varchar(13) default NULL, `publicationyear` text, @@ -730,7 +730,7 @@ CREATE TABLE `deletedbiblioitems` ( -- information about bibliographic records t `volume` mediumtext, `number` mediumtext, `itemtype` varchar(10) default NULL, -- biblio level item type (MARC21 942$c) - `isbn` varchar(30) default NULL, -- ISBN (MARC21 020$a) + `isbn` varchar(255) default NULL, -- ISBN (MARC21 020$a) `issn` varchar(9) default NULL, -- ISSN (MARC21 022$a) `ean` varchar(13) default NULL, `publicationyear` text, @@ -1048,7 +1048,7 @@ CREATE TABLE `import_biblios` ( `original_source` varchar(25) default NULL, `title` varchar(128) default NULL, `author` varchar(80) default NULL, - `isbn` varchar(30) default NULL, + `isbn` varchar(255) default NULL, `issn` varchar(9) default NULL, `has_items` tinyint(1) NOT NULL default 0, CONSTRAINT `import_biblios_ibfk_1` FOREIGN KEY (`import_record_id`) @@ -2095,7 +2095,7 @@ CREATE TABLE `suggestions` ( -- purchase suggestions `volumedesc` varchar(255) default NULL, `publicationyear` smallint(6) default 0, `place` varchar(255) default NULL, -- publication place of the suggested item - `isbn` varchar(30) default NULL, -- isbn of the suggested item + `isbn` varchar(255) default NULL, -- isbn of the suggested item `mailoverseeing` smallint(1) default 0, `biblionumber` int(11) default NULL, -- foreign key linking the suggestion to the biblio table after the suggestion has been ordered `reason` text, -- reason for accepting or rejecting the suggestion @@ -3030,7 +3030,7 @@ CREATE TABLE `biblioimages` ( DROP TABLE IF EXISTS `social_data`; CREATE TABLE IF NOT EXISTS `social_data` ( - `isbn` VARCHAR(30), + `isbn` VARCHAR(255), `num_critics` INT, `num_critics_pro` INT, `num_quotations` INT, diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index 5963d24..f1f2a9f 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -7010,6 +7010,18 @@ CREATE TABLE IF NOT EXISTS borrower_files ( SetVersion($DBversion); } +$DBversion = "3.13.00.XXX"; +if ( CheckVersion($DBversion) ) { + $dbh->do("ALTER TABLE biblioitems CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); + $dbh->do("ALTER TABLE deletedbiblioitems CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); + $dbh->do("ALTER TABLE import_biblios CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); + $dbh->do("ALTER TABLE suggestions CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); + $dbh->do("ALTER TABLE social_data CHANGE isbn isbn VARCHAR( 255 )"); + + print "Upgrade to $DBversion done (Improve ISBN matching)\n"; + SetVersion($DBversion); +} + =head1 FUNCTIONS =head2 TableExists($table) diff --git a/t/db_dependent/Biblio.t b/t/db_dependent/Biblio.t index 4725e53..4a470f3 100755 --- a/t/db_dependent/Biblio.t +++ b/t/db_dependent/Biblio.t @@ -13,12 +13,18 @@ BEGIN { use_ok('C4::Biblio'); } -my $isbn = '0590353403'; +my @isbns = ('0491001304', '0914378260 (pbk. : v. 1) :', '0394502884 (Random House) :', 'a0877790019 (black leather)', '0456789012 (reel 1)' ); my $title = 'Foundation'; -my $marc_record=MARC::Record->new; -my $field = MARC::Field->new('020','','','a' => $isbn); -$marc_record->append_fields($field); +my $marc_record = MARC::Record->new; +my $field; + +foreach my $isbn ( @isbns ) { + $field = MARC::Field->new('020','','','a' => $isbn); + $marc_record->append_fields($field); +} +my $isbns_string = C4::Biblio::GetNormalizedISBNsFromMarc( $marc_record ); + my($biblionumber,$biblioitemnumber) = AddBiblio($marc_record,''); my $data = &GetBiblioData($biblionumber); is($data->{Title},undef,'Makes sure title field in biblio is empty.'); @@ -28,11 +34,11 @@ $marc_record->append_fields($field); ModBiblio($marc_record,$biblionumber,''); $data = &GetBiblioData($biblionumber); is($data->{title},$title,'uses ModBiblio to add a title to the previously created record and checks that its there.'); -is($data->{isbn},$isbn,'Makes sure the isbn is still there after using ModBiblio.'); +is($data->{isbn},$isbns_string,'Makes sure the isbn is still there after using ModBiblio.'); my $itemdata = &GetBiblioItemData($biblioitemnumber); is($itemdata->{title},$title,'First test of GetBiblioItemData to get same result of previous two GetBiblioData tests.'); -is($itemdata->{isbn},$isbn,'Second test checking it returns the correct isbn.'); +is($itemdata->{isbn},$isbns_string,'Second test checking it returns the correct isbn.'); my $success = 0; $field = MARC::Field->new( -- 1.7.2.5