View | Details | Raw Unified | Return to bug 4283
Collapse All | Expand All

(-)a/C4/Biblio.pm (+37 lines)
Lines 64-69 BEGIN { Link Here
64
      &GetBiblioItemByBiblioNumber
64
      &GetBiblioItemByBiblioNumber
65
      &GetBiblioFromItemNumber
65
      &GetBiblioFromItemNumber
66
      &GetBiblionumberFromItemnumber
66
      &GetBiblionumberFromItemnumber
67
      &GetBiblionumberSlice
67
68
68
      &GetRecordValue
69
      &GetRecordValue
69
      &GetFieldMapping
70
      &GetFieldMapping
Lines 874-879 sub GetBiblionumberFromItemnumber { Link Here
874
    return ($result);
875
    return ($result);
875
}
876
}
876
877
878
=head2 GetBiblionumberSlice
879
880
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice( 100, 450 ); #Get 100 biblionumbers after skipping 450 oldest biblionumbers.
881
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice( 100, undef, 110004347 ); #Get 100 biblionumbers after biblionumber 110004347
882
883
@PARAM1 Long, maximum amount of biblio-rows to return. Same as the SQL LIMIT-clause.
884
              Defaults to 0.
885
@PARAM2 Long, how many biblio-rows to skip starting from the first row. Same as the SQL OFFSET-clause.
886
              Defaults to 500.
887
@PARAM3 Long, the biblionumber (inclusive) from which to start (ascending) getting the slice. Overrides @PARAM2.
888
@RETURN Array of Long, a slice of biblionumbers starting from the offset and no more rows than the limit-parameter.
889
=cut
890
891
sub GetBiblionumberSlice {
892
    my ($limit, $offset, $biblionumber) = @_;
893
    $limit = ($limit) ? $limit : 500 ;
894
    $offset = ($offset) ? $offset : 0;
895
896
    my $dbh            = C4::Context->dbh;
897
    my $sth;
898
    if ($biblionumber) {
899
        $sth = $dbh->prepare("SELECT biblionumber FROM biblio WHERE biblionumber >= ? LIMIT ?");
900
        $sth->execute($biblionumber, $limit);
901
    }
902
    else {
903
        $sth = $dbh->prepare("SELECT biblionumber FROM biblio LIMIT ? OFFSET ?");
904
        $sth->execute($limit, $offset);
905
    }
906
907
    my @biblionumbers;
908
    while(my $bn = $sth->fetchrow()) {
909
        push @biblionumbers, $bn;
910
    }
911
    return \@biblionumbers;
912
}
913
877
=head2 GetBiblioFromItemNumber
914
=head2 GetBiblioFromItemNumber
878
915
879
  $item = &GetBiblioFromItemNumber($itemnumber,$barcode);
916
  $item = &GetBiblioFromItemNumber($itemnumber,$barcode);
(-)a/cataloguing/deduplicator.pl (+131 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
4
# Copyright 2009 BibLibre
5
# Parts Copyright Catalyst IT 2011
6
#
7
# This file is part of Koha.
8
#
9
# Koha is free software; you can redistribute it and/or modify it under the
10
# terms of the GNU General Public License as published by the Free Software
11
# Foundation; either version 2 of the License, or (at your option) any later
12
# version.
13
#
14
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
15
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
16
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
17
#
18
# You should have received a copy of the GNU General Public License along
19
# with Koha; if not, write to the Free Software Foundation, Inc.,
20
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
use Modern::Perl;
23
use CGI;
24
use C4::Output;
25
use C4::Auth;
26
27
use C4::Matcher;
28
use C4::Items;
29
use C4::Biblio;
30
31
32
my $input = new CGI;
33
34
my ( $template, $loggedinuser, $cookie ) = get_template_and_user(
35
    {
36
        template_name   => "cataloguing/deduplicator.tt",
37
        query           => $input,
38
        type            => "intranet",
39
        authnotrequired => 0,
40
        flagsrequired   => { editcatalogue => 'edit_catalogue' },
41
    }
42
);
43
44
my $max_matches = 100;
45
my $matcher_id = $input->param('matcher_id');
46
my $op = $input->param('op');
47
my $param_limit = $input->param('limit');
48
$template->param(   limit => $param_limit   ) if $param_limit;
49
my $param_offset = $input->param('offset');
50
$template->param(   offset => $param_offset   ) if $param_offset;
51
my $param_biblionumber = $input->param('biblionumber');
52
$template->param(   biblionumber => $param_biblionumber   ) if $param_biblionumber;
53
54
55
#Get the matchers list and set the selected matcher as selected.
56
my @matchers = C4::Matcher::GetMatcherList( $matcher_id );
57
foreach (@matchers) {
58
    if ($matcher_id && $_->{matcher_id} == $matcher_id) {
59
        $_->{selected} = 1;
60
        last();
61
    }
62
}
63
$template->param(   matchers => \@matchers   );
64
65
if ($op && $op eq 'deduplicate') {
66
    my $matcher = C4::Matcher->fetch($matcher_id);
67
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $param_limit, $param_offset, $param_biblionumber );
68
69
    my @duplicates;
70
    foreach my $biblionumber (@$biblionumbers) {
71
        my $marc = C4::Biblio::GetMarcBiblio($biblionumber);
72
        my @matches = $matcher->get_matches( $marc, $max_matches );
73
74
        if (scalar(@matches) > 1) {
75
            foreach my $match (@matches) {
76
                my $itemsCount = C4::Items::GetItemsCount($match->{record_id});
77
                $match->{itemsCount} = $itemsCount;
78
                buildSlimBiblio($biblionumber, $match, $marc);
79
                if ($match->{record_id} == $biblionumber) {
80
                    $match->{matchSource} = 'matchSource';
81
                }
82
                
83
            }
84
            my $biblio = buildSlimBiblio($biblionumber, undef, $marc);
85
            $biblio->{matches} = \@matches;
86
87
            push @duplicates, $biblio;
88
        }
89
    }
90
    $template->param(   duplicates => \@duplicates   ) if scalar(@duplicates) > 0;
91
}
92
93
output_html_with_http_headers $input, $cookie, $template->output;
94
95
96
sub buildSlimBiblio {
97
    my ($biblionumber, $biblio, $marc) = @_;
98
99
    if ($biblio) {
100
        $biblio->{biblionumber} = $biblionumber;
101
    }
102
    else {
103
        $biblio = {biblionumber => $biblionumber};
104
    }
105
106
    my $title = $marc->subfield('245','a');
107
    my $titleField;
108
    my @titles;
109
    if ($title) {
110
        $titleField = '245';
111
    }
112
    else {
113
        $titleField = '240';
114
        $title = $marc->subfield('240','a');
115
    }
116
    my $enumeration = $marc->subfield( $titleField ,'n');
117
    my $partName = $marc->subfield( $titleField ,'p');
118
    my $publicationYear = $marc->subfield( '260' ,'c');
119
    push @titles, $title if $title;
120
    push @titles, $enumeration if $enumeration;
121
    push @titles, $partName if $partName;
122
    push @titles, $publicationYear if $publicationYear;
123
124
    my $author = $marc->subfield('100','a');
125
    $author = $marc->subfield('110','a') unless $author;
126
127
    $biblio->{author} = $author;
128
    $biblio->{title} = join(' ', @titles);
129
130
    return $biblio;
131
}
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbooks.tt (+3 lines)
Lines 73-78 Link Here
73
                [% END %]
73
                [% END %]
74
            </ul>
74
            </ul>
75
        </div>
75
        </div>
76
        <div class="btn-group">
77
            <a href="/cgi-bin/koha/cataloguing/deduplicator.pl"><button class="btn btn-small">Deduplicator</button></a>
78
        </div>
76
  </div>
79
  </div>
77
[% END %]
80
[% END %]
78
81
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/deduplicator.tt (+156 lines)
Line 0 Link Here
1
[% INCLUDE 'doc-head-open.inc' %]
2
<title>Koha &rsaquo; Cataloging &rsaquo; Deduplicator</title>
3
[% INCLUDE 'greybox.inc' %]
4
[% INCLUDE 'doc-head-close.inc' %]
5
<style type="text/css">
6
    /* Prevent floating the main form */
7
    fieldset.rows { float:none; }
8
    
9
    .mergeReference { color: #161; font-weight: bold; }
10
    .cell { float: left; padding-left: 1em; }
11
    .matchSource { background-color: #f1e8f6; }
12
    input[type=checkbox] { vertical-align: bottom; }
13
    
14
    #duplicatesContainer li { padding-bottom: 0px; margin-bottom: 0em; }
15
    #duplicatesContainer li:last-of-type { padding-bottom: 0px; margin-bottom: 1em; }
16
    #duplicatesContainer li.sourceContainer { padding-bottom: 0px; margin-bottom: 1em; }
17
</style>
18
<script type="text/javascript">
19
//<![CDATA[
20
21
    function updateMergereference(self) {
22
        var matchContainer = $(self).parents(".matchContainer");
23
        var checkboxes = $(matchContainer).find('input[id^="checkbox"]');
24
        var mergeReferenceInput = $(matchContainer).parents("form").children("input.mergeReference");
25
        var checkedBoxes = 0;
26
        $(checkboxes).each(function() {
27
            if($(this).attr('checked') === 'checked' ) {
28
                checkedBoxes++;
29
            }
30
        });
31
32
        if (checkedBoxes == 1  &&  $(self).attr('checked') === 'checked') {
33
            $(self).parent().find("span.matchDescription").addClass('mergeReference');
34
            $(mergeReferenceInput).val( $(self).val() ); //Save the biblionumber as the mergereference
35
        }
36
        else if (checkedBoxes == 0) {
37
            $(matchContainer).find("span.matchDescription").removeClass('mergeReference');
38
            $(mergeReferenceInput).val( 0 );
39
        }
40
41
        return 1;
42
    }
43
//]]>
44
</script>
45
</head>
46
<body id="deduplicator">
47
[% INCLUDE 'header.inc' %]
48
[% INCLUDE 'cataloging-search.inc' %]
49
<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> &rsaquo; <a href="/cgi-bin/koha/cataloguing/addbooks.pl">Cataloging</a>  &rsaquo; Deduplicator</div>
50
51
<div id="doc" class="yui-t7">
52
<div id="bd">
53
    <div id="yui-main">
54
55
56
        <h1>Deduplicator</h1>
57
58
        <div id="deduplicatorForm">
59
            <form name="deduplicationForm" action="/cgi-bin/koha/cataloguing/deduplicator.pl" method="post">
60
                <input type="hidden" name="op" value="deduplicate"/>
61
                <fieldset class="rows">
62
                    <legend>Configure the deduplication search!</legend>
63
                    <ol>
64
                        <li>
65
                            <label>Limit</label><input type="number" name="limit" value="[% limit %]"/><i>How many bibliographic records to deduplicate? Hint: less than 1000.</i>
66
                        </li>
67
                        <li>
68
                            <label>Offset</label><input type="number" name="offset" value="[% offset %]"/><i>How many bibliographic records to skip from the start? Hint: increment this by the limit of previous deduplication run. If limit = 200, then first offset is 0, then 200, then 400... This is useful for manual deduplication of the whole database.</i>
69
                        </li>
70
                        <li>
71
                            <label>Biblionumber</label><input type="number" name="biblionumber" value="[% biblionumber %]"/><i>From which bibliographic record to start? Obsoletes "Offset". Hint: If you know for certain that after a given biblionumber you have several duplicates, it is easier to target those new duplicate entries using this.</i>
72
                        </li>
73
                        <li>
74
                            <label>Matcher</label>
75
                            <select name="matcher_id" size="1" id="matchers">
76
                            [% FOREACH matcher IN matchers %]
77
                                [% IF ( matcher.selected ) %]
78
                                    <option value="[% matcher.matcher_id %]" selected="selected">[% matcher.code %] - [% matcher.description %]</option>
79
                                [% ELSE %]
80
                                    <option value="[% matcher.matcher_id %]">[% matcher.code %] - [% matcher.description %]</option>
81
                                [% END %]
82
                            [% END %]
83
                            </select>
84
                            <i>The matcher to use to find duplicates. Hint: one can <a href="/cgi-bin/koha/admin/matching-rules.pl">define matchers</a> from the administration menu. For more information see the Koha manual at <a href="koha-community.org/documentation/">koha-community.org/documentation/</a></i>
85
                        </li>
86
                        <li>
87
                            <input type="submit" value="Deduplicate!"/>
88
                        </li>
89
                    </ol>
90
                </fieldset>
91
            </form>
92
        </div>
93
94
95
96
[% IF duplicates %]
97
        <p>
98
            <i>
99
                -To deduplicate a found match group, first select the biblio to merge to. The target of the merge is colored. Then select the biblios to merge to the colored target. To reset the merge target, simply remove all checks under one match group.<br/>
100
                -The original match source is distinguished from the list of matches by a beautiful pastel color.<br/>
101
                -Each matched row starts with the biblionumber, separated by a dash, then the matching score configurable in the Matcher, then the amount of items this biblio has. The named identifier consists of the title, enumeration, part name, publication year and author.<br/>
102
                -All merge requests always open to a new tab.
103
            </i>
104
        </p>
105
        <div id="duplicatesContainer">
106
            <fieldset class="rows">
107
                <legend>
108
                    List of duplicates
109
                </legend>
110
111
                <!--<label>Matches - Score - Items count</label>
112
                <label>Title</label>
113
                <label>Author</label>-->
114
115
                <ol>
116
                [% FOREACH duplicate IN duplicates %]
117
                    <form target="_blank" name="form[% duplicate.biblionumber %]" action="/cgi-bin/koha/cataloguing/merge.pl" method="post">
118
                        <input type="hidden" class="mergeReference" name="mergereference" value=""/>
119
                        <li class="sourceContainer">
120
                            <b>
121
                                <a href="/cgi-bin/koha/catalogue/showmarc.pl?id=[% duplicate.biblionumber %]">[% duplicate.biblionumber %]</a>
122
                                [% duplicate.title %] &nbsp;
123
                                [% duplicate.author %]
124
                            </b>
125
                            <input type="submit" value="Merge!"/>
126
                        </li>
127
                        <span class="matchContainer">
128
                            [% FOREACH match IN duplicate.matches %]
129
                                <li>
130
                                    <span  class="[% match.matchSource %]">
131
                                    <input id="checkbox[% match.record_id %]" name="biblionumber" value="[% match.record_id %]" type="checkbox" onclick="updateMergereference(this)"/>
132
                                    <a href="/cgi-bin/koha/catalogue/showmarc.pl?id=[% match.record_id %]">
133
                                        <span class="matchDescription">
134
                                            [% match.record_id %] - <i>[% match.score %]</i>
135
                                        </span>
136
                                    </a>
137
                                    <a href="/cgi-bin/koha/catalogue/detail.pl?biblionumber=[% match.record_id %]">
138
                                        &nbsp;[[% match.itemsCount %]]
139
                                    </a>
140
                                    [% match.title %] &nbsp;
141
                                    [% match.author %]
142
                                    </span>
143
                                </li>
144
                            [% END %]
145
                        </span>
146
                    </form>
147
                [% END %]
148
                </ol>
149
            </fieldset>
150
        </div><!--<div id="duplicatesContainer">%-->
151
[% END %]
152
    </div>
153
</div> <!--EO <div id="bd"> -->
154
155
156
[% INCLUDE 'intranet-bottom.inc' %]
(-)a/t/db_dependent/Biblio.t (-4 / +84 lines)
Lines 150-155 sub run_tests { Link Here
150
    is( scalar @$issns, 4,
150
    is( scalar @$issns, 4,
151
        'GetMARCISSN skips empty ISSN fields (Bug 12674)');
151
        'GetMARCISSN skips empty ISSN fields (Bug 12674)');
152
152
153
    testGetBiblionumberSlice($marcflavour);
154
153
    ## Testing GetMarcControlnumber
155
    ## Testing GetMarcControlnumber
154
    my $controlnumber;
156
    my $controlnumber;
155
    $controlnumber = GetMarcControlnumber( $marc_record, $marcflavour );
157
    $controlnumber = GetMarcControlnumber( $marc_record, $marcflavour );
Lines 229-234 sub mock_marcfromkohafield { Link Here
229
        });
231
        });
230
}
232
}
231
233
234
sub addMockBiblio {
235
    my $isbn = shift;
236
    my $marcflavour = shift;
237
238
    # Generate a record with just the ISBN
239
    my $marc_record = MARC::Record->new;
240
    my $isbn_field  = create_isbn_field( $isbn, $marcflavour );
241
    $marc_record->append_fields( $isbn_field );
242
243
    # Add the record to the DB
244
    my ( $biblionumber, $biblioitemnumber ) = AddBiblio( $marc_record, '' );
245
    return ( $biblionumber, $biblioitemnumber );
246
}
247
232
sub create_title_field {
248
sub create_title_field {
233
    my ( $title, $marcflavour ) = @_;
249
    my ( $title, $marcflavour ) = @_;
234
250
Lines 260-281 sub create_issn_field { Link Here
260
}
276
}
261
277
262
subtest 'MARC21' => sub {
278
subtest 'MARC21' => sub {
263
    plan tests => 27;
279
    plan tests => 33;
264
    run_tests('MARC21');
280
    run_tests('MARC21');
265
    $dbh->rollback;
281
    $dbh->rollback;
266
};
282
};
267
283
268
subtest 'UNIMARC' => sub {
284
subtest 'UNIMARC' => sub {
269
    plan tests => 27;
285
    plan tests => 33;
270
    run_tests('UNIMARC');
286
    run_tests('UNIMARC');
271
    $dbh->rollback;
287
    $dbh->rollback;
272
};
288
};
273
289
274
subtest 'NORMARC' => sub {
290
subtest 'NORMARC' => sub {
275
    plan tests => 27;
291
    plan tests => 33;
276
    run_tests('NORMARC');
292
    run_tests('NORMARC');
277
    $dbh->rollback;
293
    $dbh->rollback;
278
};
294
};
279
295
296
##Testing C4::Biblio::GetBiblionumberSlice(), runs 6 tests
297
sub testGetBiblionumberSlice() {
298
    my $marcflavour = shift;
299
300
    #Get all biblionumbers.
301
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice(999999999999);
302
    my $initialCount = scalar(@$biblionumbers);
303
    is( ($initialCount > 0), 1, 'C4::Biblio::GetBiblionumberSlice(), Get all biblionumbers.');
304
305
    #Add a bunch of mock biblios.
306
    my ($bn1) = addMockBiblio('0120344506', $marcflavour);
307
    my ($bn2) = addMockBiblio('0230455607', $marcflavour);
308
    my ($bn3) = addMockBiblio('0340566708', $marcflavour);
309
    my ($bn4) = addMockBiblio('0450677809', $marcflavour);
310
    my ($bn5) = addMockBiblio('0560788900', $marcflavour);
311
312
    #Get all biblionumbers again, but now we should have 5 more.
313
    $biblionumbers = C4::Biblio::GetBiblionumberSlice(999999999999);
314
    is( $initialCount+5, scalar(@$biblionumbers), 'C4::Biblio::GetBiblionumberSlice(), Get all biblionumbers after appending 5 biblios more.');
315
316
    #Get 3 biblionumbers.
317
    $biblionumbers = C4::Biblio::GetBiblionumberSlice(3);
318
    is( 3, scalar(@$biblionumbers), 'C4::Biblio::GetBiblionumberSlice(), Get 3 biblionumbers.');
319
320
    #Get 3 biblionumbers, all of whom must be of the recently added.
321
    $biblionumbers = C4::Biblio::GetBiblionumberSlice(3, $initialCount);
322
    my $testOK = 1;
323
    foreach (@$biblionumbers) {
324
        if ($_ == $bn1 || $_ == $bn2 || $_ == $bn3) {
325
            #The result is part of us!
326
        }
327
        else {
328
            $testOK = 0; #Fail the test because we got some biblionumbers we were not meant to get.
329
        }
330
    }
331
    is( $testOK, 1, 'C4::Biblio::GetBiblionumberSlice(), Get 3 specific biblionumbers.');
332
333
    #Get 3 biblionumbers, all of whom must be $bn3 or added right after it.
334
    $biblionumbers = C4::Biblio::GetBiblionumberSlice(3, undef, $bn3);
335
    $testOK = 1;
336
    foreach (@$biblionumbers) {
337
        if ($_ == $bn3 || $_ == $bn4 || $_ == $bn5) {
338
            #The result is part of us!
339
        }
340
        else {
341
            $testOK = 0; #Fail the test because we got some biblionumbers we were not meant to get.
342
        }
343
    }
344
    is( $testOK, 1, 'C4::Biblio::GetBiblionumberSlice(), Get 3 specific biblionumbers after a specific biblionumber.');
345
346
    #Same test as the previous one, but test for offset-parameter overriding by the biblionumber-parameter.
347
    $biblionumbers = C4::Biblio::GetBiblionumberSlice(3, $initialCount, $bn3);
348
    $testOK = 1;
349
    foreach (@$biblionumbers) {
350
        if ($_ == $bn3 || $_ == $bn4 || $_ == $bn5) {
351
            #The result is part of us!
352
        }
353
        else {
354
            #Fail the test because we got some biblionumbers we were not meant to get.
355
            #These biblionumbers are probably $bn1 and $bn2.
356
            $testOK = 0;
357
        }
358
    }
359
    is( $testOK, 1, 'C4::Biblio::GetBiblionumberSlice(), offset-parameter overriding.');
360
}
280
361
281
1;
362
1;
282
- 

Return to bug 4283