From 29e3741c3997fc5d6f084828490f36f1ed17334c Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Fri, 28 Nov 2014 14:18:32 +0200 Subject: [PATCH] Bug 4283 - Merge bibliographic records Adds a GUI for deduplicating regions of the biblionumber space in Koha. Deduplication happens using the given C4::Matcher. Deduplicated biblionumber slice is limited with LIMIT- and OFFSET-clauses as well as a starting biblionumber by C4::Biblio::GetBiblionumberSlice(). Deduplication search has the following defaults: Limit = 500, Offset = 0, Matcher = the lowest matcher_id in DB. The speed of the deduplication process is mostly influenced by the complexity of the Matcher, but the GUI can handle a limit of ~1000 relatively well with a Matcher using ~7 match points taking a couple of minutes. From the deduplication GUI, merging is done with the lists-based cataloguing/merge.pl -tool. -!-!-!-!-!-!-!-!-!- -TEST PREPARATION:- -!-!-!-!-!-!-!-!-!- -2. Define/choose a Matcher from the admin/matching-rules.pl. For tutorial purposes use the default ISBN-based Matcher. -1. Add two or more Bibliographic records, each having the same ISBN. Note the biblionumbers. 0. Reindex the search index. -!-!-!-!-!-!- -TEST PLAN:- -!-!-!-!-!-!- 1. Go to cataloguing home (cataloguing/addbooks.pl) and observe a new toolbar item, "Deduplicator"! Go there! 2. In the deduplicator.pl, choose your Matcher and "Deduplicate!". This uses the default values. 2.a. If you don't get any results, enter the smaller biblionumber of the recently added duplicate ISBN Biblios' to the input field. Then retry submission. Use Limit of 2 for convenience. 3. Click the checkbox of a matched result to set it as the "merge target/reference", then click one or more matches to merge with the "merge target/reference", then click the "Merge!"-button on the boldfaced "match source"-row. 4. Merge using the merge.pl-tool as you normally would. As of writing that merge-feature supports only two Biblios, but Buug 8064 will bring support for merging multiple Biblios at once. UNIT TESTS included for the C4::Biblio::GetBiblionumberSlice() --- C4/Biblio.pm | 37 +++++ cataloguing/deduplicator.pl | 131 ++++++++++++++++ .../prog/en/modules/cataloguing/addbooks.tt | 3 + .../prog/en/modules/cataloguing/deduplicator.tt | 156 ++++++++++++++++++++ t/db_dependent/Biblio.t | 87 ++++++++++- 5 files changed, 411 insertions(+), 3 deletions(-) create mode 100755 cataloguing/deduplicator.pl create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/deduplicator.tt diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 8bdb25f..4eb590b 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -64,6 +64,7 @@ BEGIN { &GetBiblioItemByBiblioNumber &GetBiblioFromItemNumber &GetBiblionumberFromItemnumber + &GetBiblionumberSlice &GetRecordValue &GetFieldMapping @@ -874,6 +875,42 @@ sub GetBiblionumberFromItemnumber { return ($result); } +=head2 GetBiblionumberSlice + + my $biblionumbers = C4::Biblio::GetBiblionumberSlice( 100, 450 ); #Get 100 biblionumbers after skipping 450 oldest biblionumbers. + my $biblionumbers = C4::Biblio::GetBiblionumberSlice( 100, undef, 110004347 ); #Get 100 biblionumbers after biblionumber 110004347 + +@PARAM1 Long, maximum amount of biblio-rows to return. Same as the SQL LIMIT-clause. + Defaults to 0. +@PARAM2 Long, how many biblio-rows to skip starting from the first row. Same as the SQL OFFSET-clause. + Defaults to 500. +@PARAM3 Long, the biblionumber (inclusive) from which to start (ascending) getting the slice. Overrides @PARAM2. +@RETURN Array of Long, a slice of biblionumbers starting from the offset and no more rows than the limit-parameter. +=cut + +sub GetBiblionumberSlice { + my ($limit, $offset, $biblionumber) = @_; + $limit = ($limit) ? $limit : 500 ; + $offset = ($offset) ? $offset : 0; + + my $dbh = C4::Context->dbh; + my $sth; + if ($biblionumber) { + $sth = $dbh->prepare("SELECT biblionumber FROM biblio WHERE biblionumber >= ? LIMIT ?"); + $sth->execute($biblionumber, $limit); + } + else { + $sth = $dbh->prepare("SELECT biblionumber FROM biblio LIMIT ? OFFSET ?"); + $sth->execute($limit, $offset); + } + + my @biblionumbers; + while(my $bn = $sth->fetchrow()) { + push @biblionumbers, $bn; + } + return \@biblionumbers; +} + =head2 GetBiblioFromItemNumber $item = &GetBiblioFromItemNumber($itemnumber,$barcode); diff --git a/cataloguing/deduplicator.pl b/cataloguing/deduplicator.pl new file mode 100755 index 0000000..11603f0 --- /dev/null +++ b/cataloguing/deduplicator.pl @@ -0,0 +1,131 @@ +#!/usr/bin/perl + + +# Copyright 2009 BibLibre +# Parts Copyright Catalyst IT 2011 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use CGI; +use C4::Output; +use C4::Auth; + +use C4::Matcher; +use C4::Items; +use C4::Biblio; + + +my $input = new CGI; + +my ( $template, $loggedinuser, $cookie ) = get_template_and_user( + { + template_name => "cataloguing/deduplicator.tt", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => { editcatalogue => 'edit_catalogue' }, + } +); + +my $max_matches = 100; +my $matcher_id = $input->param('matcher_id'); +my $op = $input->param('op'); +my $param_limit = $input->param('limit'); +$template->param( limit => $param_limit ) if $param_limit; +my $param_offset = $input->param('offset'); +$template->param( offset => $param_offset ) if $param_offset; +my $param_biblionumber = $input->param('biblionumber'); +$template->param( biblionumber => $param_biblionumber ) if $param_biblionumber; + + +#Get the matchers list and set the selected matcher as selected. +my @matchers = C4::Matcher::GetMatcherList( $matcher_id ); +foreach (@matchers) { + if ($matcher_id && $_->{matcher_id} == $matcher_id) { + $_->{selected} = 1; + last(); + } +} +$template->param( matchers => \@matchers ); + +if ($op && $op eq 'deduplicate') { + my $matcher = C4::Matcher->fetch($matcher_id); + my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $param_limit, $param_offset, $param_biblionumber ); + + my @duplicates; + foreach my $biblionumber (@$biblionumbers) { + my $marc = C4::Biblio::GetMarcBiblio($biblionumber); + my @matches = $matcher->get_matches( $marc, $max_matches ); + + if (scalar(@matches) > 1) { + foreach my $match (@matches) { + my $itemsCount = C4::Items::GetItemsCount($match->{record_id}); + $match->{itemsCount} = $itemsCount; + buildSlimBiblio($biblionumber, $match, $marc); + if ($match->{record_id} == $biblionumber) { + $match->{matchSource} = 'matchSource'; + } + + } + my $biblio = buildSlimBiblio($biblionumber, undef, $marc); + $biblio->{matches} = \@matches; + + push @duplicates, $biblio; + } + } + $template->param( duplicates => \@duplicates ) if scalar(@duplicates) > 0; +} + +output_html_with_http_headers $input, $cookie, $template->output; + + +sub buildSlimBiblio { + my ($biblionumber, $biblio, $marc) = @_; + + if ($biblio) { + $biblio->{biblionumber} = $biblionumber; + } + else { + $biblio = {biblionumber => $biblionumber}; + } + + my $title = $marc->subfield('245','a'); + my $titleField; + my @titles; + if ($title) { + $titleField = '245'; + } + else { + $titleField = '240'; + $title = $marc->subfield('240','a'); + } + my $enumeration = $marc->subfield( $titleField ,'n'); + my $partName = $marc->subfield( $titleField ,'p'); + my $publicationYear = $marc->subfield( '260' ,'c'); + push @titles, $title if $title; + push @titles, $enumeration if $enumeration; + push @titles, $partName if $partName; + push @titles, $publicationYear if $publicationYear; + + my $author = $marc->subfield('100','a'); + $author = $marc->subfield('110','a') unless $author; + + $biblio->{author} = $author; + $biblio->{title} = join(' ', @titles); + + return $biblio; +} \ No newline at end of file diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbooks.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbooks.tt index 8a02dd5..6eae94d 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbooks.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbooks.tt @@ -73,6 +73,9 @@ [% END %] +
+ +
[% END %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/deduplicator.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/deduplicator.tt new file mode 100644 index 0000000..28eb2fb --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/deduplicator.tt @@ -0,0 +1,156 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Cataloging › Deduplicator +[% INCLUDE 'greybox.inc' %] +[% INCLUDE 'doc-head-close.inc' %] + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cataloging-search.inc' %] + + +
+
+
+ + +

Deduplicator

+ +
+
+ +
+ Configure the deduplication search! +
    +
  1. + How many bibliographic records to deduplicate? Hint: less than 1000. +
  2. +
  3. + How many bibliographic records to skip from the start? Hint: increment this by the limit of previous deduplication run. If limit = 200, then first offset is 0, then 200, then 400... This is useful for manual deduplication of the whole database. +
  4. +
  5. + From which bibliographic record to start? Obsoletes "Offset". Hint: If you know for certain that after a given biblionumber you have several duplicates, it is easier to target those new duplicate entries using this. +
  6. +
  7. + + + The matcher to use to find duplicates. Hint: one can define matchers from the administration menu. For more information see the Koha manual at koha-community.org/documentation/ +
  8. +
  9. + +
  10. +
+
+
+
+ + + +[% IF duplicates %] +

+ + -To deduplicate a found match group, first select the biblio to merge to. The target of the merge is colored. Then select the biblios to merge to the colored target. To reset the merge target, simply remove all checks under one match group.
+ -The original match source is distinguished from the list of matches by a beautiful pastel color.
+ -Each matched row starts with the biblionumber, separated by a dash, then the matching score configurable in the Matcher, then the amount of items this biblio has. The named identifier consists of the title, enumeration, part name, publication year and author.
+ -All merge requests always open to a new tab. +
+

+
+
+ + List of duplicates + + + + +
    + [% FOREACH duplicate IN duplicates %] +
    + +
  1. + + [% duplicate.biblionumber %] + [% duplicate.title %]   + [% duplicate.author %] + + +
  2. + + [% FOREACH match IN duplicate.matches %] +
  3. + + + + + [% match.record_id %] - [% match.score %] + + + +  [[% match.itemsCount %]] + + [% match.title %]   + [% match.author %] + +
  4. + [% END %] +
    +
    + [% END %] +
+
+
+[% END %] +
+
+ + +[% INCLUDE 'intranet-bottom.inc' %] \ No newline at end of file diff --git a/t/db_dependent/Biblio.t b/t/db_dependent/Biblio.t index f53b6a5..1fd0088 100755 --- a/t/db_dependent/Biblio.t +++ b/t/db_dependent/Biblio.t @@ -150,6 +150,8 @@ sub run_tests { is( scalar @$issns, 4, 'GetMARCISSN skips empty ISSN fields (Bug 12674)'); + testGetBiblionumberSlice($marcflavour); + ## Testing GetMarcControlnumber my $controlnumber; $controlnumber = GetMarcControlnumber( $marc_record, $marcflavour ); @@ -229,6 +231,20 @@ sub mock_marcfromkohafield { }); } +sub addMockBiblio { + my $isbn = shift; + my $marcflavour = shift; + + # Generate a record with just the ISBN + my $marc_record = MARC::Record->new; + my $isbn_field = create_isbn_field( $isbn, $marcflavour ); + $marc_record->append_fields( $isbn_field ); + + # Add the record to the DB + my ( $biblionumber, $biblioitemnumber ) = AddBiblio( $marc_record, '' ); + return ( $biblionumber, $biblioitemnumber ); +} + sub create_title_field { my ( $title, $marcflavour ) = @_; @@ -260,22 +276,87 @@ sub create_issn_field { } subtest 'MARC21' => sub { - plan tests => 27; + plan tests => 33; run_tests('MARC21'); $dbh->rollback; }; subtest 'UNIMARC' => sub { - plan tests => 27; + plan tests => 33; run_tests('UNIMARC'); $dbh->rollback; }; subtest 'NORMARC' => sub { - plan tests => 27; + plan tests => 33; run_tests('NORMARC'); $dbh->rollback; }; +##Testing C4::Biblio::GetBiblionumberSlice(), runs 6 tests +sub testGetBiblionumberSlice() { + my $marcflavour = shift; + + #Get all biblionumbers. + my $biblionumbers = C4::Biblio::GetBiblionumberSlice(999999999999); + my $initialCount = scalar(@$biblionumbers); + is( ($initialCount > 0), 1, 'C4::Biblio::GetBiblionumberSlice(), Get all biblionumbers.'); + + #Add a bunch of mock biblios. + my ($bn1) = addMockBiblio('0120344506', $marcflavour); + my ($bn2) = addMockBiblio('0230455607', $marcflavour); + my ($bn3) = addMockBiblio('0340566708', $marcflavour); + my ($bn4) = addMockBiblio('0450677809', $marcflavour); + my ($bn5) = addMockBiblio('0560788900', $marcflavour); + + #Get all biblionumbers again, but now we should have 5 more. + $biblionumbers = C4::Biblio::GetBiblionumberSlice(999999999999); + is( $initialCount+5, scalar(@$biblionumbers), 'C4::Biblio::GetBiblionumberSlice(), Get all biblionumbers after appending 5 biblios more.'); + + #Get 3 biblionumbers. + $biblionumbers = C4::Biblio::GetBiblionumberSlice(3); + is( 3, scalar(@$biblionumbers), 'C4::Biblio::GetBiblionumberSlice(), Get 3 biblionumbers.'); + + #Get 3 biblionumbers, all of whom must be of the recently added. + $biblionumbers = C4::Biblio::GetBiblionumberSlice(3, $initialCount); + my $testOK = 1; + foreach (@$biblionumbers) { + if ($_ == $bn1 || $_ == $bn2 || $_ == $bn3) { + #The result is part of us! + } + else { + $testOK = 0; #Fail the test because we got some biblionumbers we were not meant to get. + } + } + is( $testOK, 1, 'C4::Biblio::GetBiblionumberSlice(), Get 3 specific biblionumbers.'); + + #Get 3 biblionumbers, all of whom must be $bn3 or added right after it. + $biblionumbers = C4::Biblio::GetBiblionumberSlice(3, undef, $bn3); + $testOK = 1; + foreach (@$biblionumbers) { + if ($_ == $bn3 || $_ == $bn4 || $_ == $bn5) { + #The result is part of us! + } + else { + $testOK = 0; #Fail the test because we got some biblionumbers we were not meant to get. + } + } + is( $testOK, 1, 'C4::Biblio::GetBiblionumberSlice(), Get 3 specific biblionumbers after a specific biblionumber.'); + + #Same test as the previous one, but test for offset-parameter overriding by the biblionumber-parameter. + $biblionumbers = C4::Biblio::GetBiblionumberSlice(3, $initialCount, $bn3); + $testOK = 1; + foreach (@$biblionumbers) { + if ($_ == $bn3 || $_ == $bn4 || $_ == $bn5) { + #The result is part of us! + } + else { + #Fail the test because we got some biblionumbers we were not meant to get. + #These biblionumbers are probably $bn1 and $bn2. + $testOK = 0; + } + } + is( $testOK, 1, 'C4::Biblio::GetBiblionumberSlice(), offset-parameter overriding.'); +} 1; -- 1.7.9.5