Bugzilla – Attachment 19172 Details for
Bug 10500
Improve isbn matching when importing records
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
WIP - Improve ISBN Matching
WIP---Improve-ISBN-Matching.patch (text/plain), 11.87 KB, created by
Kyle M Hall (khall)
on 2013-06-20 14:26:58 UTC
(
hide
)
Description:
WIP - Improve ISBN Matching
Filename:
MIME Type:
Creator:
Kyle M Hall (khall)
Created:
2013-06-20 14:26:58 UTC
Size:
11.87 KB
patch
obsolete
>From 7374fdc42b2dc743ed28ac94a192fee57406dd9f Mon Sep 17 00:00:00 2001 >From: Kyle M Hall <kyle@bywatersolutions.com> >Date: Wed, 19 Jun 2013 12:25:02 -0400 >Subject: [PATCH] WIP - Improve ISBN Matching > >http://bugs.koha-community.org/show_bug.cgi?id=10500 >--- > C4/Biblio.pm | 131 ++++++++++++++++++++++++++++++++ > C4/ImportBatch.pm | 3 +- > C4/Matcher.pm | 8 ++- > installer/data/mysql/kohastructure.sql | 10 +- > installer/data/mysql/updatedatabase.pl | 12 +++ > t/db_dependent/Biblio.t | 18 +++-- > 6 files changed, 167 insertions(+), 15 deletions(-) > >diff --git a/C4/Biblio.pm b/C4/Biblio.pm >index 7af8619..2017fac 100644 >--- a/C4/Biblio.pm >+++ b/C4/Biblio.pm >@@ -29,6 +29,7 @@ use MARC::File::USMARC; > use MARC::File::XML; > use POSIX qw(strftime); > use Module::Load::Conditional qw(can_load); >+use Business::ISBN; > > use C4::Koha; > use C4::Dates qw/format_date/; >@@ -73,6 +74,10 @@ BEGIN { > &GetMarcControlnumber > &GetMarcNotes > &GetMarcISBN >+ &GetVariationsOfISBNs >+ &GetVariationsOfISBN >+ &GetNormalizedISBNsFromMarc >+ &NormalizeISBN > &GetMarcISSN > &GetMarcSubjects > &GetMarcBiblio >@@ -1663,6 +1668,130 @@ sub GetMarcISBN { > } # end GetMarcISBN > > >+=head2 GetNormalizedISBNsFromMarc >+ >+ my @isbns = GetNormalizedISBNsFromMarc( $record ); >+ >+ Returns a list of normalized ISBNs from a given record >+ or, in a scalar context, returns a string of normalized >+ isbns separated by ' | ' >+ >+=cut >+ >+sub GetNormalizedISBNsFromMarc { >+ my ($record) = @_; >+ my ( $marc_field, $marc_subfield ) = >+ GetMarcFromKohaField('biblioitems.isbn'); >+ >+ my @isbns; >+ foreach my $subfield ( $record->field( $marc_field, $marc_subfield ) ) { >+ my $isbn = NormalizeISBN( { isbn => $subfield->as_string } ); >+ push( @isbns, $isbn ) if ($isbn); >+ } >+ >+ # remove duplicate isbns >+ @isbns = keys %{ { map { $_ => 1 } @isbns } }; >+ >+ return wantarray ? @isbns : join( " | ", @isbns ); >+} >+ >+ >+=head2 GetVariationsOfISBN >+ >+ my @isbns = GetVariationsOfISBN( $isbn ); >+ >+ Returns a list of varations of the given isbn in >+ both ISBN-10 and ISBN-13 formats, with and without >+ hyphens. >+ >+ In a scalar context, the isbns are returned as a >+ string delimited by ' | '. >+ >+=cut >+ >+sub GetVariationsOfISBN { >+ my ($isbn) = @_; >+ >+ my @isbns; >+ >+ push( @isbns, NormalizeISBN({ isbn => $isbn }) ); >+ push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-10' }) ); >+ push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-13' }) ); >+ push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-10', strip_hyphens => 1 }) ); >+ push( @isbns, NormalizeISBN({ isbn => $isbn, format => 'ISBN-13', strip_hyphens => 1 }) ); >+ >+ return wantarray ? @isbns : join( " | ", @isbns ); >+} >+ >+=head2 GetVariationsOfISBNs >+ >+ my @isbns = GetVariationsOfISBNs( @isbns ); >+ >+ Returns a list of varations of the given isbns in >+ both ISBN-10 and ISBN-13 formats, with and without >+ hyphens. >+ >+ In a scalar context, the isbns are returned as a >+ string delimited by ' | '. >+ >+=cut >+ >+sub GetVariationsOfISBNs { >+ my (@isbns) = @_; >+ >+ @isbns = map { GetVariationsOfISBN( $_ ) } @isbns; >+ >+ # remove duplicate isbns >+ @isbns = keys %{ { map { $_ => 1 } @isbns } }; >+ >+ return wantarray ? @isbns : join( " | ", @isbns ); >+} >+ >+=head2 NormalizedISBN >+ >+ my $isbns = NormalizedISBN({ >+ isbn => $isbn, >+ strip_hyphens => [0,1], >+ format => ['ISBN-10', 'ISBN-13'] >+ }); >+ >+ Returns an isbn validated by Business::ISBN. >+ Optionally strips hyphens and/or forces the isbn >+ to be of the specified format. >+ >+ If the string cannot be validated as an isbn, >+ it returns nothing. >+ >+=cut >+ >+sub NormalizeISBN { >+ my ($params) = @_; >+ >+ my $string = $params->{isbn}; >+ my $strip_hyphens = $params->{strip_hyphens}; >+ my $format = $params->{format}; >+ >+ my $isbn = Business::ISBN->new($string); >+ >+ if ( $isbn && $isbn->error != Business::ISBN::BAD_ISBN ) { >+ >+ if ( $format eq 'ISBN-10' ) { >+ $isbn = $isbn->as_isbn10(); >+ } >+ elsif ( $format eq 'ISBN-13' ) { >+ $isbn = $isbn->as_isbn13(); >+ } >+ >+ if ($strip_hyphens) { >+ $string = $isbn->as_string( [] ); >+ } else { >+ $string = $isbn->as_string(); >+ } >+ >+ return $string; >+ } >+} >+ > =head2 GetMarcISSN > > $marcissnsarray = GetMarcISSN( $record, $marcflavour ); >@@ -2569,6 +2698,8 @@ sub TransformMarcToKoha { > } > } > >+ $result->{'isbn'} = GetNormalizedISBNsFromMarc( $record ); >+ > return $result; > } > >diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm >index 3ce5c67..5f2c2f3 100644 >--- a/C4/ImportBatch.pm >+++ b/C4/ImportBatch.pm >@@ -323,7 +323,7 @@ sub ModAuthInBatch { > > =cut > >-sub BatchStageMarcRecords { >+sub BatchStageMarcRecords { > my $record_type = shift; > my $encoding = shift; > my $marc_records = shift; >@@ -1399,7 +1399,6 @@ sub _add_biblio_fields { > my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record); > my $dbh = C4::Context->dbh; > # FIXME no controlnumber, originalsource >- $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public > my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)"); > $sth->execute($import_record_id, $title, $author, $isbn, $issn); > $sth->finish(); >diff --git a/C4/Matcher.pm b/C4/Matcher.pm >index 4e064be..b407a55 100644 >--- a/C4/Matcher.pm >+++ b/C4/Matcher.pm >@@ -630,8 +630,11 @@ sub get_matches { > $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser')); > foreach my $matchpoint ( @{ $self->{'matchpoints'} } ) { > my @source_keys = _get_match_keys( $source_record, $matchpoint ); >+ > next if scalar(@source_keys) == 0; > >+ @source_keys = C4::Biblio::GetVariationsOfISBNs( @source_keys ) if ( $matchpoint->{index} eq 'isbn' ); >+ > # build query > my $query; > my $error; >@@ -640,13 +643,14 @@ sub get_matches { > if ( $self->{'record_type'} eq 'biblio' ) { > if ($QParser) { > $query = join( " || ", >- map { "$matchpoint->{'index'}:$_" } @source_keys ); >+ map { "$matchpoint->{'index'},phr:$_" } @source_keys ); > } > else { > $query = join( " or ", >- map { "$matchpoint->{'index'}=$_" } @source_keys ); >+ map { "$matchpoint->{'index'},phr=$_" } @source_keys ); > } > require C4::Search; >+ > ( $error, $searchresults, $total_hits ) = > C4::Search::SimpleSearch( $query, 0, $max_matches ); > } >diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql >index d7eebdd..1ca4373 100644 >--- a/installer/data/mysql/kohastructure.sql >+++ b/installer/data/mysql/kohastructure.sql >@@ -152,7 +152,7 @@ CREATE TABLE `biblioitems` ( -- information related to bibliographic records in > `volume` mediumtext, > `number` mediumtext, > `itemtype` varchar(10) default NULL, -- biblio level item type (MARC21 942$c) >- `isbn` varchar(30) default NULL, -- ISBN (MARC21 020$a) >+ `isbn` varchar(255) default NULL, -- ISBN (MARC21 020$a) > `issn` varchar(9) default NULL, -- ISSN (MARC21 022$a) > `ean` varchar(13) default NULL, > `publicationyear` text, >@@ -730,7 +730,7 @@ CREATE TABLE `deletedbiblioitems` ( -- information about bibliographic records t > `volume` mediumtext, > `number` mediumtext, > `itemtype` varchar(10) default NULL, -- biblio level item type (MARC21 942$c) >- `isbn` varchar(30) default NULL, -- ISBN (MARC21 020$a) >+ `isbn` varchar(255) default NULL, -- ISBN (MARC21 020$a) > `issn` varchar(9) default NULL, -- ISSN (MARC21 022$a) > `ean` varchar(13) default NULL, > `publicationyear` text, >@@ -1048,7 +1048,7 @@ CREATE TABLE `import_biblios` ( > `original_source` varchar(25) default NULL, > `title` varchar(128) default NULL, > `author` varchar(80) default NULL, >- `isbn` varchar(30) default NULL, >+ `isbn` varchar(255) default NULL, > `issn` varchar(9) default NULL, > `has_items` tinyint(1) NOT NULL default 0, > CONSTRAINT `import_biblios_ibfk_1` FOREIGN KEY (`import_record_id`) >@@ -2095,7 +2095,7 @@ CREATE TABLE `suggestions` ( -- purchase suggestions > `volumedesc` varchar(255) default NULL, > `publicationyear` smallint(6) default 0, > `place` varchar(255) default NULL, -- publication place of the suggested item >- `isbn` varchar(30) default NULL, -- isbn of the suggested item >+ `isbn` varchar(255) default NULL, -- isbn of the suggested item > `mailoverseeing` smallint(1) default 0, > `biblionumber` int(11) default NULL, -- foreign key linking the suggestion to the biblio table after the suggestion has been ordered > `reason` text, -- reason for accepting or rejecting the suggestion >@@ -3030,7 +3030,7 @@ CREATE TABLE `biblioimages` ( > > DROP TABLE IF EXISTS `social_data`; > CREATE TABLE IF NOT EXISTS `social_data` ( >- `isbn` VARCHAR(30), >+ `isbn` VARCHAR(255), > `num_critics` INT, > `num_critics_pro` INT, > `num_quotations` INT, >diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl >index 5963d24..f1f2a9f 100755 >--- a/installer/data/mysql/updatedatabase.pl >+++ b/installer/data/mysql/updatedatabase.pl >@@ -7010,6 +7010,18 @@ CREATE TABLE IF NOT EXISTS borrower_files ( > SetVersion($DBversion); > } > >+$DBversion = "3.13.00.XXX"; >+if ( CheckVersion($DBversion) ) { >+ $dbh->do("ALTER TABLE biblioitems CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); >+ $dbh->do("ALTER TABLE deletedbiblioitems CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); >+ $dbh->do("ALTER TABLE import_biblios CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); >+ $dbh->do("ALTER TABLE suggestions CHANGE isbn isbn VARCHAR( 255 ) NULL DEFAULT NULL"); >+ $dbh->do("ALTER TABLE social_data CHANGE isbn isbn VARCHAR( 255 )"); >+ >+ print "Upgrade to $DBversion done (Improve ISBN matching)\n"; >+ SetVersion($DBversion); >+} >+ > =head1 FUNCTIONS > > =head2 TableExists($table) >diff --git a/t/db_dependent/Biblio.t b/t/db_dependent/Biblio.t >index 4725e53..4a470f3 100755 >--- a/t/db_dependent/Biblio.t >+++ b/t/db_dependent/Biblio.t >@@ -13,12 +13,18 @@ BEGIN { > use_ok('C4::Biblio'); > } > >-my $isbn = '0590353403'; >+my @isbns = ('0491001304', '0914378260 (pbk. : v. 1) :', '0394502884 (Random House) :', 'a0877790019 (black leather)', '0456789012 (reel 1)' ); > my $title = 'Foundation'; > >-my $marc_record=MARC::Record->new; >-my $field = MARC::Field->new('020','','','a' => $isbn); >-$marc_record->append_fields($field); >+my $marc_record = MARC::Record->new; >+my $field; >+ >+foreach my $isbn ( @isbns ) { >+ $field = MARC::Field->new('020','','','a' => $isbn); >+ $marc_record->append_fields($field); >+} >+my $isbns_string = C4::Biblio::GetNormalizedISBNsFromMarc( $marc_record ); >+ > my($biblionumber,$biblioitemnumber) = AddBiblio($marc_record,''); > my $data = &GetBiblioData($biblionumber); > is($data->{Title},undef,'Makes sure title field in biblio is empty.'); >@@ -28,11 +34,11 @@ $marc_record->append_fields($field); > ModBiblio($marc_record,$biblionumber,''); > $data = &GetBiblioData($biblionumber); > is($data->{title},$title,'uses ModBiblio to add a title to the previously created record and checks that its there.'); >-is($data->{isbn},$isbn,'Makes sure the isbn is still there after using ModBiblio.'); >+is($data->{isbn},$isbns_string,'Makes sure the isbn is still there after using ModBiblio.'); > > my $itemdata = &GetBiblioItemData($biblioitemnumber); > is($itemdata->{title},$title,'First test of GetBiblioItemData to get same result of previous two GetBiblioData tests.'); >-is($itemdata->{isbn},$isbn,'Second test checking it returns the correct isbn.'); >+is($itemdata->{isbn},$isbns_string,'Second test checking it returns the correct isbn.'); > > my $success = 0; > $field = MARC::Field->new( >-- >1.7.2.5
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 10500
:
19172
|
19174
|
20732
|
20744
|
21072
|
21257
|
21258
|
22840
|
22841
|
22910
|
22911
|
22918
|
22919
|
27220
|
27221
|
27222
|
27387
|
27388
|
27389