Attachment #34509 for bug #4283

View | Details | Raw Unified | Return to bug 4283
Collapse All | Expand All




    my $verbose = $self->{verbose};
    my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $self->{limit}, $self->{offset}, $self->{biblionumber} );

    $self->{duplicates} = [];
    foreach my $biblionumber (@$biblionumbers) {
        my $marc = C4::Biblio::GetMarcBiblio($biblionumber);
        my @matches = $self->{matcher}->get_matches( $marc, $self->{max_matches} );

        if (scalar(@matches) > 1) {
            for (my $i=0 ; $i<scalar(@matches) ; $i++) {
                my $match = $matches[$i];
                my $itemsCount = C4::Items::GetItemsCount($match->{record_id});
                $match->{itemsCount} = $itemsCount;
                unless(  _buildSlimBiblio($match->{record_id}, $match, C4::Biblio::GetMarcBiblio($match->{record_id}))  ) {
                    #Sometimes we get an error where the marcxml is not available.
                    splice(@matches, $i, 1);
                    $i--; #Don't advance the iterator after this round or we will skip one record!
                    next();
                }
                if ($match->{record_id} == $biblionumber) {
                    $match->{matchSource} = 'matchSource';
                }
                
            }
            my $biblio = _buildSlimBiblio($biblionumber, undef, $marc);
            unless ($biblio) { #Sometimes we get an error where the marcxml is not available.
                next();
            }
            $biblio->{matches} = \@matches;

            push @{$self->{duplicates}}, $biblio;
        }
        if ($verbose) {
            print $biblionumber."\n";
        }
    }
    return $self->{duplicates};
}

sub _buildSlimBiblio {

    }
    if (not($marc)) {
        warn "C4::Deduplicator::_buildSlimBiblio(), No MARC::Record for bn:$biblionumber";
        return undef;
    }
    
    $biblio->{marc} = $marc;

    my $author = $marc->subfield('100','a');
    $author = $marc->subfield('110','a') unless $author;

    $biblio->{author} = ($author) ? $author : '';
    $biblio->{title} = join(' ', @titles);
    $biblio->{title} = '' unless $biblio->{title};

    return $biblio;
}

sub batchMergeDuplicates {
    my ($self, $duplicates, $mergeTargetFindingAlgorithm) = @_;

    $self->{mergeErrors} = [];
    _findMergeTargets($duplicates, $mergeTargetFindingAlgorithm, $self->{mergeErrors});

    foreach my $duplicate (@$duplicates) {
        foreach my $match (@{$duplicate->{matches}}) {
            if ($match eq $duplicate->{'mergeTarget'}) { #Comparing Perl references, if htey point to the same object.
                next(); #Don't merge itself to oneself.
            }
            merge($match, $duplicate->{'mergeTarget'}, $self->{mergeErrors});
        }
    }
    return $self->{mergeErrors} if scalar @{$self->{mergeErrors}} > 0;
    return undef;
}


    if ($mergeTargetFindingAlgorithm eq 'newest') {
        _mergeTargetFindingAlgorithm_newest( $duplicates );
    }
    else {
        warn "Unknown merge target finding algorithm given: '$mergeTargetFindingAlgorithm'";
    }
}

sub _mergeTargetFindingAlgorithm_newest {

    my $dbh = C4::Context->dbh;
    my $sth;

    # Creating a new record from the html code
    my $tobiblio     =  $mergeTarget->{biblionumber};
    my $frombiblio   =  $match->{biblionumber};
    if ($tobiblio == $frombiblio) {

        push @$errors, $error if ($error);
    }
}

sub printDuplicatesAsText {
    my ($self) = @_;

    foreach my $duplicate (@{$self->{duplicates}}) {
        print 'Match source: '.$duplicate->{biblionumber}.' - '.$duplicate->{title}.' '.$duplicate->{author}."\n";
        foreach my $match (@{$duplicate->{matches}}) {
            print $match->{record_id}.' - '.$match->{score}.' '.$match->{itemsCount}.'  '.$match->{title}.' '.$match->{author}."\n";
        }
        print "\n\n";
    }
}

sub printMergesAsText {
    my ($self) = @_;
    foreach my $error (@{$self->{mergeErrors}}) {
        print $error;
    }
}
1;

Lines 26-38 use Koha::Deduplicator; Link Here

(-)a/misc/maintenance/deduplicator.pl (-22 / +56 lines)
26		26
27	use Getopt::Long qw(:config no_ignore_case);	27	use Getopt::Long qw(:config no_ignore_case);
28		28
29	my ($help, $verbose, $offset, $biblionumber, $matcher_id, $merge);	29	my ($help, $verbose, $biblionumber, $matcher_id, $merge);
30	my $limit = 500;	30	my $limit = 500;
		31	my $chunk = 500;
		32	my $offset = 0;
31		33
32	GetOptions(	34	GetOptions(
33	'h\|help' => \$help,	35	'h\|help' => \$help,
34	'v\|verbose' => \$verbose,	36	'v\|verbose' => \$verbose,
35	'l\|limit:i' => \$limit,	37	'l\|limit:i' => \$limit,
		38	'c\|chunk:i' => \$chunk,
36	'o\|offset:i' => \$offset,	39	'o\|offset:i' => \$offset,
37	'b\|biblionumber:i' => \$biblionumber,	40	'b\|biblionumber:i' => \$biblionumber,
38	'm\|matcher:i' => \$matcher_id,	41	'm\|matcher:i' => \$matcher_id,
Lines 52-60 This script has the following parameters : Link Here
52		55
53	-l --limit How many biblios to check for duplicates. Is the SQL	56	-l --limit How many biblios to check for duplicates. Is the SQL
54	LIMIT-clause for gathering biblios to deduplicate.	57	LIMIT-clause for gathering biblios to deduplicate.
		58	Defaults to 500. To run through the whole DB, set a sufficiently
		59	large number, like 999999999999999 :)
		60
		61	-c --chunk How many records to process on one deduplicate->merge run.
		62	Defaults to 500. Use this to prevent memory from running
		63	out when deduplicating/merging large databases.
55		64
56	-o --offset How many records to skip from the start. Is the SQL	65	-o --offset How many records to skip from the start. Is the SQL
57	OFFSET-clause for gathering biblios to deduplicate.	66	OFFSET-clause for gathering biblios to deduplicate.
		67	Defaults to 0.
58		68
59	-b --biblionumber From which biblionumber (inclusive) to start gathering	69	-b --biblionumber From which biblionumber (inclusive) to start gathering
60	the biblios to deduplicate. Obsoletes --offset	70	the biblios to deduplicate. Obsoletes --offset
Lines 94-123 elsif ($merge) { Link Here
94	exit;	104	exit;
95	}	105	}
96		106
97	my ($deduplicator, $initErrors) = Koha::Deduplicator->new( $matcher_id, $limit, $offset, $biblionumber, $verbose );	107	my $lastOffset = $offset;
98	if ($initErrors) {	108	while ($lastOffset < $limit) {
99	print "Errors happened when creating the Deduplicator:\n";	109
100	print join("\n", @$initErrors);	110	my $chunkSize = _calculateChunkSize($lastOffset, $chunk, $limit);
101	print "\n";	111
102	print $usage;	112	runDeduplicateMergeChunk($matcher_id, $chunkSize, $lastOffset, $biblionumber, $verbose );
103	exit;	113
		114	$lastOffset += $chunk;
104	}	115	}
105	else {
106	my $duplicates = $deduplicator->deduplicate();
107		116
108	foreach my $duplicate (@$duplicates) {	117	=head _calculateChunkSize
109	print 'Match source: '.$duplicate->{biblionumber}.' - '.$duplicate->{title}.' '.$duplicate->{author}."\n";	118
110	foreach my $match (@{$duplicate->{matches}}) {	119	my $chunkSize = _calculateChunkSize($lastOffset, $chunk, $limit);
111	print $match->{record_id}.' - '.$match->{score}.' '.$match->{itemsCount}.' '.$match->{title}.' '.$match->{author}."\n";	120
112	}	121	It can be that the last chunk overflows the limit-paramter, thus leading to deduplicating/merging too many biblios.
113	print "\n\n";	122	We don't want that, so calculate the remaining chunk size to not exceed the given limit!
		123	=cut
		124	sub _calculateChunkSize {
		125	my ($lastOffset, $chunk, $limit) = @_;
		126
		127	my $chunkSize = $chunk;
		128	if ($lastOffset + $chunk > $limit) {
		129	$chunkSize = $limit - $lastOffset;
114	}	130	}
		131	return $chunkSize;
		132	}
		133
		134	sub runDeduplicateMergeChunk {
		135	my ($matcher_id, $chunkSize, $offset, $biblionumber, $verbose ) = @_;
		136
		137	my ($deduplicator, $initErrors) = Koha::Deduplicator->new( $matcher_id, $chunkSize, $offset, $biblionumber, $verbose );
		138	if ($initErrors) {
		139	print "Errors happened when creating the Deduplicator:\n";
		140	print join("\n", @$initErrors);
		141	print "\n";
		142	print $usage;
		143	exit;
		144	}
		145	else {
		146	my $duplicates = $deduplicator->deduplicate();
		147
		148	$deduplicator->printDuplicatesAsText();
115		149
116	if ($merge && $duplicates) {	150	if ($merge && scalar(@$duplicates) > 0) {
117	my $errors = $deduplicator->batchMergeDuplicates($duplicates, $merge);	151	my $errors = $deduplicator->batchMergeDuplicates($duplicates, $merge);
118	if ($errors) {	152	if ($errors) {
119	foreach my $error (@$errors) {	153	foreach my $error (@$errors) {
120	print $error;	154	print $error;
		155	}
121	}	156	}
122	}	157	}
123	}	158	}
124	-

Return to bug 4283

Lines 89-119 sub deduplicate { Link Here

(-)a/Koha/Deduplicator.pm (-13 / +43 lines)
89	my $verbose = $self->{verbose};	89	my $verbose = $self->{verbose};
90	my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $self->{limit}, $self->{offset}, $self->{biblionumber} );	90	my $biblionumbers = C4::Biblio::GetBiblionumberSlice( $self->{limit}, $self->{offset}, $self->{biblionumber} );
91		91
92	my @duplicates;	92	$self->{duplicates} = [];
93	foreach my $biblionumber (@$biblionumbers) {	93	foreach my $biblionumber (@$biblionumbers) {
94	my $marc = C4::Biblio::GetMarcBiblio($biblionumber);	94	my $marc = C4::Biblio::GetMarcBiblio($biblionumber);
95	my @matches = $self->{matcher}->get_matches( $marc, $self->{max_matches} );	95	my @matches = $self->{matcher}->get_matches( $marc, $self->{max_matches} );
96		96
97	if (scalar(@matches) > 1) {	97	if (scalar(@matches) > 1) {
98	foreach my $match (@matches) {	98	for (my $i=0 ; $i<scalar(@matches) ; $i++) {
		99	my $match = $matches[$i];
99	my $itemsCount = C4::Items::GetItemsCount($match->{record_id});	100	my $itemsCount = C4::Items::GetItemsCount($match->{record_id});
100	$match->{itemsCount} = $itemsCount;	101	$match->{itemsCount} = $itemsCount;
101	_buildSlimBiblio($match->{record_id}, $match, C4::Biblio::GetMarcBiblio($match->{record_id}));	102	unless( _buildSlimBiblio($match->{record_id}, $match, C4::Biblio::GetMarcBiblio($match->{record_id})) ) {
		103	#Sometimes we get an error where the marcxml is not available.
		104	splice(@matches, $i, 1);
		105	$i--; #Don't advance the iterator after this round or we will skip one record!
		106	next();
		107	}
102	if ($match->{record_id} == $biblionumber) {	108	if ($match->{record_id} == $biblionumber) {
103	$match->{matchSource} = 'matchSource';	109	$match->{matchSource} = 'matchSource';
104	}	110	}
105
106	}	111	}
107	my $biblio = _buildSlimBiblio($biblionumber, undef, $marc);	112	my $biblio = _buildSlimBiblio($biblionumber, undef, $marc);
		113	unless ($biblio) { #Sometimes we get an error where the marcxml is not available.
		114	next();
		115	}
108	$biblio->{matches} = \@matches;	116	$biblio->{matches} = \@matches;
109		117
110	push @duplicates, $biblio;	118	push @{$self->{duplicates}}, $biblio;
111	}	119	}
112	if ($verbose) {	120	if ($verbose) {
113	print $biblionumber."\n";	121	print $biblionumber."\n";
114	}	122	}
115	}	123	}
116	return \@duplicates;	124	return $self->{duplicates};
117	}	125	}
118		126
119	sub _buildSlimBiblio {	127	sub _buildSlimBiblio {
Lines 127-133 sub _buildSlimBiblio { Link Here
127	}	135	}
128	if (not($marc)) {	136	if (not($marc)) {
129	warn "C4::Deduplicator::_buildSlimBiblio(), No MARC::Record for bn:$biblionumber";	137	warn "C4::Deduplicator::_buildSlimBiblio(), No MARC::Record for bn:$biblionumber";
130	return $biblio;	138	return undef;
131	}	139	}
132		140
133	$biblio->{marc} = $marc;	141	$biblio->{marc} = $marc;
Lines 153-160 sub _buildSlimBiblio { Link Here
153	my $author = $marc->subfield('100','a');	161	my $author = $marc->subfield('100','a');
154	$author = $marc->subfield('110','a') unless $author;	162	$author = $marc->subfield('110','a') unless $author;
155		163
156	$biblio->{author} = $author;	164	$biblio->{author} = ($author) ? $author : '';
157	$biblio->{title} = join(' ', @titles);	165	$biblio->{title} = join(' ', @titles);
		166	$biblio->{title} = '' unless $biblio->{title};
158		167
159	return $biblio;	168	return $biblio;
160	}	169	}
Lines 167-184 sub _buildSlimBiblio { Link Here
167	sub batchMergeDuplicates {	176	sub batchMergeDuplicates {
168	my ($self, $duplicates, $mergeTargetFindingAlgorithm) = @_;	177	my ($self, $duplicates, $mergeTargetFindingAlgorithm) = @_;
169		178
170	my @errors;	179	$self->{mergeErrors} = [];
171	_findMergeTargets($duplicates, $mergeTargetFindingAlgorithm, \@errors);	180	_findMergeTargets($duplicates, $mergeTargetFindingAlgorithm, $self->{mergeErrors});
172		181
173	foreach my $duplicate (@$duplicates) {	182	foreach my $duplicate (@$duplicates) {
174	foreach my $match (@{$duplicate->{matches}}) {	183	foreach my $match (@{$duplicate->{matches}}) {
175	if ($match eq $duplicate->{'mergeTarget'}) { #Comparing Perl references, if htey point to the same object.	184	if ($match eq $duplicate->{'mergeTarget'}) { #Comparing Perl references, if htey point to the same object.
176	next(); #Don't merge itself to oneself.	185	next(); #Don't merge itself to oneself.
177	}	186	}
178	merge($match, $duplicate->{'mergeTarget'}, \@errors);	187	merge($match, $duplicate->{'mergeTarget'}, $self->{mergeErrors});
179	}	188	}
180	}	189	}
181	return \@errors if scalar @errors > 0;	190	return $self->{mergeErrors} if scalar @{$self->{mergeErrors}} > 0;
182	return undef;	191	return undef;
183	}	192	}
184		193
Lines 188-193 sub _findMergeTargets { Link Here
188	if ($mergeTargetFindingAlgorithm eq 'newest') {	197	if ($mergeTargetFindingAlgorithm eq 'newest') {
189	_mergeTargetFindingAlgorithm_newest( $duplicates );	198	_mergeTargetFindingAlgorithm_newest( $duplicates );
190	}	199	}
		200	else {
		201	warn "Unknown merge target finding algorithm given: '$mergeTargetFindingAlgorithm'";
		202	}
191	}	203	}
192		204
193	sub _mergeTargetFindingAlgorithm_newest {	205	sub _mergeTargetFindingAlgorithm_newest {
Lines 224-230 sub merge { Link Here
224	my $dbh = C4::Context->dbh;	236	my $dbh = C4::Context->dbh;
225	my $sth;	237	my $sth;
226		238
227	# Creating a new record from the html code
228	my $tobiblio = $mergeTarget->{biblionumber};	239	my $tobiblio = $mergeTarget->{biblionumber};
229	my $frombiblio = $match->{biblionumber};	240	my $frombiblio = $match->{biblionumber};
230	if ($tobiblio == $frombiblio) {	241	if ($tobiblio == $frombiblio) {
Lines 286-289 sub merge { Link Here
286	push @$errors, $error if ($error);	297	push @$errors, $error if ($error);
287	}	298	}
288	}	299	}
		300
		301	sub printDuplicatesAsText {
		302	my ($self) = @_;
		303
		304	foreach my $duplicate (@{$self->{duplicates}}) {
		305	print 'Match source: '.$duplicate->{biblionumber}.' - '.$duplicate->{title}.' '.$duplicate->{author}."\n";
		306	foreach my $match (@{$duplicate->{matches}}) {
		307	print $match->{record_id}.' - '.$match->{score}.' '.$match->{itemsCount}.' '.$match->{title}.' '.$match->{author}."\n";
		308	}
		309	print "\n\n";
		310	}
		311	}
		312
		313	sub printMergesAsText {
		314	my ($self) = @_;
		315	foreach my $error (@{$self->{mergeErrors}}) {
		316	print $error;
		317	}
		318	}
289	1;	319	1;