View | Details | Raw Unified | Return to bug 4283
Collapse All | Expand All

(-)a/Koha/Deduplicator.pm (-13 / +43 lines)
Lines 89-119 sub deduplicate { Link Here
89
    my $verbose = $self->{verbose};
89
    my $verbose = $self->{verbose};
90
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $self->{limit}, $self->{offset}, $self->{biblionumber} );
90
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $self->{limit}, $self->{offset}, $self->{biblionumber} );
91
91
92
    my @duplicates;
92
    $self->{duplicates} = [];
93
    foreach my $biblionumber (@$biblionumbers) {
93
    foreach my $biblionumber (@$biblionumbers) {
94
        my $marc = C4::Biblio::GetMarcBiblio($biblionumber);
94
        my $marc = C4::Biblio::GetMarcBiblio($biblionumber);
95
        my @matches = $self->{matcher}->get_matches( $marc, $self->{max_matches} );
95
        my @matches = $self->{matcher}->get_matches( $marc, $self->{max_matches} );
96
96
97
        if (scalar(@matches) > 1) {
97
        if (scalar(@matches) > 1) {
98
            foreach my $match (@matches) {
98
            for (my $i=0 ; $i<scalar(@matches) ; $i++) {
99
                my $match = $matches[$i];
99
                my $itemsCount = C4::Items::GetItemsCount($match->{record_id});
100
                my $itemsCount = C4::Items::GetItemsCount($match->{record_id});
100
                $match->{itemsCount} = $itemsCount;
101
                $match->{itemsCount} = $itemsCount;
101
                _buildSlimBiblio($match->{record_id}, $match, C4::Biblio::GetMarcBiblio($match->{record_id}));
102
                unless(  _buildSlimBiblio($match->{record_id}, $match, C4::Biblio::GetMarcBiblio($match->{record_id}))  ) {
103
                    #Sometimes we get an error where the marcxml is not available.
104
                    splice(@matches, $i, 1);
105
                    $i--; #Don't advance the iterator after this round or we will skip one record!
106
                    next();
107
                }
102
                if ($match->{record_id} == $biblionumber) {
108
                if ($match->{record_id} == $biblionumber) {
103
                    $match->{matchSource} = 'matchSource';
109
                    $match->{matchSource} = 'matchSource';
104
                }
110
                }
105
                
106
            }
111
            }
107
            my $biblio = _buildSlimBiblio($biblionumber, undef, $marc);
112
            my $biblio = _buildSlimBiblio($biblionumber, undef, $marc);
113
            unless ($biblio) { #Sometimes we get an error where the marcxml is not available.
114
                next();
115
            }
108
            $biblio->{matches} = \@matches;
116
            $biblio->{matches} = \@matches;
109
117
110
            push @duplicates, $biblio;
118
            push @{$self->{duplicates}}, $biblio;
111
        }
119
        }
112
        if ($verbose) {
120
        if ($verbose) {
113
            print $biblionumber."\n";
121
            print $biblionumber."\n";
114
        }
122
        }
115
    }
123
    }
116
    return \@duplicates;
124
    return $self->{duplicates};
117
}
125
}
118
126
119
sub _buildSlimBiblio {
127
sub _buildSlimBiblio {
Lines 127-133 sub _buildSlimBiblio { Link Here
127
    }
135
    }
128
    if (not($marc)) {
136
    if (not($marc)) {
129
        warn "C4::Deduplicator::_buildSlimBiblio(), No MARC::Record for bn:$biblionumber";
137
        warn "C4::Deduplicator::_buildSlimBiblio(), No MARC::Record for bn:$biblionumber";
130
        return $biblio;
138
        return undef;
131
    }
139
    }
132
    
140
    
133
    $biblio->{marc} = $marc;
141
    $biblio->{marc} = $marc;
Lines 153-160 sub _buildSlimBiblio { Link Here
153
    my $author = $marc->subfield('100','a');
161
    my $author = $marc->subfield('100','a');
154
    $author = $marc->subfield('110','a') unless $author;
162
    $author = $marc->subfield('110','a') unless $author;
155
163
156
    $biblio->{author} = $author;
164
    $biblio->{author} = ($author) ? $author : '';
157
    $biblio->{title} = join(' ', @titles);
165
    $biblio->{title} = join(' ', @titles);
166
    $biblio->{title} = '' unless $biblio->{title};
158
167
159
    return $biblio;
168
    return $biblio;
160
}
169
}
Lines 167-184 sub _buildSlimBiblio { Link Here
167
sub batchMergeDuplicates {
176
sub batchMergeDuplicates {
168
    my ($self, $duplicates, $mergeTargetFindingAlgorithm) = @_;
177
    my ($self, $duplicates, $mergeTargetFindingAlgorithm) = @_;
169
178
170
    my @errors;
179
    $self->{mergeErrors} = [];
171
    _findMergeTargets($duplicates, $mergeTargetFindingAlgorithm, \@errors);
180
    _findMergeTargets($duplicates, $mergeTargetFindingAlgorithm, $self->{mergeErrors});
172
181
173
    foreach my $duplicate (@$duplicates) {
182
    foreach my $duplicate (@$duplicates) {
174
        foreach my $match (@{$duplicate->{matches}}) {
183
        foreach my $match (@{$duplicate->{matches}}) {
175
            if ($match eq $duplicate->{'mergeTarget'}) { #Comparing Perl references, if htey point to the same object.
184
            if ($match eq $duplicate->{'mergeTarget'}) { #Comparing Perl references, if htey point to the same object.
176
                next(); #Don't merge itself to oneself.
185
                next(); #Don't merge itself to oneself.
177
            }
186
            }
178
            merge($match, $duplicate->{'mergeTarget'}, \@errors);
187
            merge($match, $duplicate->{'mergeTarget'}, $self->{mergeErrors});
179
        }
188
        }
180
    }
189
    }
181
    return \@errors if scalar @errors > 0;
190
    return $self->{mergeErrors} if scalar @{$self->{mergeErrors}} > 0;
182
    return undef;
191
    return undef;
183
}
192
}
184
193
Lines 188-193 sub _findMergeTargets { Link Here
188
    if ($mergeTargetFindingAlgorithm eq 'newest') {
197
    if ($mergeTargetFindingAlgorithm eq 'newest') {
189
        _mergeTargetFindingAlgorithm_newest( $duplicates );
198
        _mergeTargetFindingAlgorithm_newest( $duplicates );
190
    }
199
    }
200
    else {
201
        warn "Unknown merge target finding algorithm given: '$mergeTargetFindingAlgorithm'";
202
    }
191
}
203
}
192
204
193
sub _mergeTargetFindingAlgorithm_newest {
205
sub _mergeTargetFindingAlgorithm_newest {
Lines 224-230 sub merge { Link Here
224
    my $dbh = C4::Context->dbh;
236
    my $dbh = C4::Context->dbh;
225
    my $sth;
237
    my $sth;
226
238
227
    # Creating a new record from the html code
228
    my $tobiblio     =  $mergeTarget->{biblionumber};
239
    my $tobiblio     =  $mergeTarget->{biblionumber};
229
    my $frombiblio   =  $match->{biblionumber};
240
    my $frombiblio   =  $match->{biblionumber};
230
    if ($tobiblio == $frombiblio) {
241
    if ($tobiblio == $frombiblio) {
Lines 286-289 sub merge { Link Here
286
        push @$errors, $error if ($error);
297
        push @$errors, $error if ($error);
287
    }
298
    }
288
}
299
}
300
301
sub printDuplicatesAsText {
302
    my ($self) = @_;
303
304
    foreach my $duplicate (@{$self->{duplicates}}) {
305
        print 'Match source: '.$duplicate->{biblionumber}.' - '.$duplicate->{title}.' '.$duplicate->{author}."\n";
306
        foreach my $match (@{$duplicate->{matches}}) {
307
            print $match->{record_id}.' - '.$match->{score}.' '.$match->{itemsCount}.'  '.$match->{title}.' '.$match->{author}."\n";
308
        }
309
        print "\n\n";
310
    }
311
}
312
313
sub printMergesAsText {
314
    my ($self) = @_;
315
    foreach my $error (@{$self->{mergeErrors}}) {
316
        print $error;
317
    }
318
}
289
1;
319
1;
(-)a/misc/maintenance/deduplicator.pl (-22 / +56 lines)
Lines 26-38 use Koha::Deduplicator; Link Here
26
26
27
use Getopt::Long qw(:config no_ignore_case);
27
use Getopt::Long qw(:config no_ignore_case);
28
28
29
my ($help, $verbose, $offset, $biblionumber, $matcher_id, $merge);
29
my ($help, $verbose, $biblionumber, $matcher_id, $merge);
30
my $limit = 500;
30
my $limit = 500;
31
my $chunk = 500;
32
my $offset = 0;
31
33
32
GetOptions(
34
GetOptions(
33
    'h|help'           => \$help,
35
    'h|help'           => \$help,
34
    'v|verbose'        => \$verbose,
36
    'v|verbose'        => \$verbose,
35
    'l|limit:i'        => \$limit,
37
    'l|limit:i'        => \$limit,
38
    'c|chunk:i'        => \$chunk,
36
    'o|offset:i'       => \$offset,
39
    'o|offset:i'       => \$offset,
37
    'b|biblionumber:i' => \$biblionumber,
40
    'b|biblionumber:i' => \$biblionumber,
38
    'm|matcher:i'      => \$matcher_id,
41
    'm|matcher:i'      => \$matcher_id,
Lines 52-60 This script has the following parameters : Link Here
52
55
53
    -l --limit        How many biblios to check for duplicates. Is the SQL
56
    -l --limit        How many biblios to check for duplicates. Is the SQL
54
                      LIMIT-clause for gathering biblios to deduplicate.
57
                      LIMIT-clause for gathering biblios to deduplicate.
58
                      Defaults to 500. To run through the whole DB, set a sufficiently
59
                      large number, like 999999999999999 :)
60
61
    -c --chunk        How many records to process on one deduplicate->merge run.
62
                      Defaults to 500. Use this to prevent memory from running
63
                      out when deduplicating/merging large databases.
55
64
56
    -o --offset       How many records to skip from the start. Is the SQL
65
    -o --offset       How many records to skip from the start. Is the SQL
57
                      OFFSET-clause for gathering biblios to deduplicate.
66
                      OFFSET-clause for gathering biblios to deduplicate.
67
                      Defaults to 0.
58
68
59
    -b --biblionumber From which biblionumber (inclusive) to start gathering
69
    -b --biblionumber From which biblionumber (inclusive) to start gathering
60
                      the biblios to deduplicate. Obsoletes --offset
70
                      the biblios to deduplicate. Obsoletes --offset
Lines 94-123 elsif ($merge) { Link Here
94
    exit;
104
    exit;
95
}
105
}
96
106
97
my ($deduplicator, $initErrors) = Koha::Deduplicator->new( $matcher_id, $limit, $offset, $biblionumber, $verbose );
107
my $lastOffset = $offset;
98
if ($initErrors) {
108
while ($lastOffset < $limit) {
99
    print "Errors happened when creating the Deduplicator:\n";
109
100
    print join("\n", @$initErrors);
110
    my $chunkSize = _calculateChunkSize($lastOffset, $chunk, $limit);
101
    print "\n";
111
102
    print $usage;
112
    runDeduplicateMergeChunk($matcher_id, $chunkSize, $lastOffset, $biblionumber, $verbose );
103
    exit;
113
114
    $lastOffset += $chunk;
104
}
115
}
105
else {
106
    my $duplicates = $deduplicator->deduplicate();
107
116
108
    foreach my $duplicate (@$duplicates) {
117
=head _calculateChunkSize
109
        print 'Match source: '.$duplicate->{biblionumber}.' - '.$duplicate->{title}.' '.$duplicate->{author}."\n";
118
110
        foreach my $match (@{$duplicate->{matches}}) {
119
    my $chunkSize = _calculateChunkSize($lastOffset, $chunk, $limit);
111
            print $match->{record_id}.' - '.$match->{score}.' '.$match->{itemsCount}.'  '.$match->{title}.' '.$match->{author}."\n";
120
112
        }
121
It can be that the last chunk overflows the limit-paramter, thus leading to deduplicating/merging too many biblios.
113
        print "\n\n";
122
We don't want that, so calculate the remaining chunk size to not exceed the given limit!
123
=cut
124
sub _calculateChunkSize {
125
    my ($lastOffset, $chunk, $limit) = @_;
126
127
    my $chunkSize = $chunk;
128
    if ($lastOffset + $chunk > $limit) {
129
        $chunkSize = $limit - $lastOffset;
114
    }
130
    }
131
    return $chunkSize;
132
}
133
134
sub runDeduplicateMergeChunk {
135
    my ($matcher_id, $chunkSize, $offset, $biblionumber, $verbose ) = @_;
136
137
    my ($deduplicator, $initErrors) = Koha::Deduplicator->new( $matcher_id, $chunkSize, $offset, $biblionumber, $verbose );
138
    if ($initErrors) {
139
        print "Errors happened when creating the Deduplicator:\n";
140
        print join("\n", @$initErrors);
141
        print "\n";
142
        print $usage;
143
        exit;
144
    }
145
    else {
146
        my $duplicates = $deduplicator->deduplicate();
147
148
        $deduplicator->printDuplicatesAsText();
115
149
116
    if ($merge && $duplicates) {
150
        if ($merge && scalar(@$duplicates) > 0) {
117
        my $errors = $deduplicator->batchMergeDuplicates($duplicates, $merge);
151
            my $errors = $deduplicator->batchMergeDuplicates($duplicates, $merge);
118
        if ($errors) {
152
            if ($errors) {
119
            foreach my $error (@$errors) {
153
                foreach my $error (@$errors) {
120
                print $error;
154
                    print $error;
155
                }
121
            }
156
            }
122
        }
157
        }
123
    }
158
    }
124
- 

Return to bug 4283