View | Details | Raw Unified | Return to bug 7286
Collapse All | Expand All

(-)a/misc/migration_tools/rebuild_zebra.pl (-1 / +22 lines)
Lines 86-91 if ( !$as_xml and $nosanitize ) { Link Here
86
    die $msg;
86
    die $msg;
87
}
87
}
88
88
89
if ( $nosanitize and $item_limit ) {
90
    my $msg = "Cannot specify both -item_limit and -nosanitize\n";
91
    $msg .= "Please do '$0 --help' to see usage.\n";
92
    die $msg;
93
}
94
89
if ($process_zebraqueue and ($skip_export or $reset)) {
95
if ($process_zebraqueue and ($skip_export or $reset)) {
90
    my $msg = "Cannot specify -r or -s if -z is specified\n";
96
    my $msg = "Cannot specify -r or -s if -z is specified\n";
91
    $msg   .= "Please do '$0 --help' to see usage.\n";
97
    $msg   .= "Please do '$0 --help' to see usage.\n";
Lines 422-427 sub generate_deleted_marc_records { Link Here
422
        my $marc = MARC::Record->new();
428
        my $marc = MARC::Record->new();
423
        if ($record_type eq 'biblio') {
429
        if ($record_type eq 'biblio') {
424
            fix_biblio_ids($marc, $record_number, $record_number);
430
            fix_biblio_ids($marc, $record_number, $record_number);
431
            fix_biblio_items( $marc ) if $item_limit;
425
        } else {
432
        } else {
426
            fix_authority_id($marc, $record_number);
433
            fix_authority_id($marc, $record_number);
427
        }
434
        }
Lines 447-453 sub get_corrected_marc_record { Link Here
447
454
448
    if (defined $marc) {
455
    if (defined $marc) {
449
        fix_leader($marc);
456
        fix_leader($marc);
450
        if ($record_type eq 'authority') {
457
        if ( $record_type eq 'biblio' ) {
458
            my $succeeded = fix_biblio_ids( $marc, $record_number );
459
            fix_biblio_items( $marc ) if $item_limit;
460
            return unless $succeeded;
461
        } else {
451
            fix_authority_id($marc, $record_number);
462
            fix_authority_id($marc, $record_number);
452
        }
463
        }
453
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
464
        if (C4::Context->preference("marcflavour") eq "UNIMARC") {
Lines 514-519 sub fix_leader { Link Here
514
    $marc->leader(substr($leader, 0, 24));
525
    $marc->leader(substr($leader, 0, 24));
515
}
526
}
516
527
528
sub fix_biblio_items {
529
    my $marc = shift;
530
    my ($itemtagfield, $itemtagsubfield) = GetMarcFromKohaField('items.itemnumber','');
531
    my $i = 0;
532
    for my $itemfield ( $marc->field($itemtagfield) ) {
533
	$marc->delete_field($itemfield) if $i >= $item_limit;
534
        $i++;
535
    }
536
}
537
517
sub fix_biblio_ids {
538
sub fix_biblio_ids {
518
    # FIXME - it is essential to ensure that the biblionumber is present,
539
    # FIXME - it is essential to ensure that the biblionumber is present,
519
    #         otherwise, Zebra will choke on the record.  However, this
540
    #         otherwise, Zebra will choke on the record.  However, this
(-)a/misc/migration_tools/rebuild_zebra_sliced.zsh (-1 / +127 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/zsh
2
3
# Copyright 2011 BibLibre SARL
4
# This file is part of Koha.
5
#
6
# Koha is free software; you can redistribute it and/or modify it under the
7
# terms of the GNU General Public License as published by the Free Software
8
# Foundation; either version 2 of the License, or (at your option) any later
9
# version.
10
#
11
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
12
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License along
16
# with Koha; if not, write to the Free Software Foundation, Inc.,
17
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18
19
20
INCREMENT=$1
21
BIBLIOSTART=$2
22
BIBLIOEND=$3
23
24
#echo " | $INCREMENT , $BIBLIOSTART , $BIBLIOEND | ";
25
# this script rebuild the zebra index recursively
26
# BIBLIOSTART is the record number to BIBLIOSTART on
27
# BIBLIOEND is the record number to BIBLIOEND on
28
# increment specify how many records we must try at once
29
# At the BIBLIOEND of each "chunk", this script checks if the indexing process has been OK
30
# if it hasn't, the slice is splitted in 10, and the reindexing is called again on each smaller chunk
31
# if the increment goes to 1, it means we tried to reindex 1 by 1, and the failing indexing concern wrong records
32
33
# the logs are stored in a directory called logs/ that must be a subdirectory of reindex.zsh
34
35
# at the BIBLIOEND of the script, just type :
36
#grep -l "previous transaction" `ls rebuild1.*.err`
37
# the result contains all the biblios that have not been indexed
38
# WARNING : the numbers are not the biblionumber but the record number, they can be reached by :
39
# SELECT biblionumber FROM biblio LIMIT YourNumberHere,1;
40
41
# EXAMPLE to run the script on a 800 000 biblios database :
42
# ./reindex.zsh 50000 0 800000
43
# will reindex the DB, BIBLIOSTARTing with chunks of 50k biblios
44
#if { grep -E "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ; then
45
46
lastbiblionumber=`perl -e '#!/usr/bin/perl
47
use C4::Context;
48
my $dbh = C4::Context->dbh;
49
my $querylastbiblionumber = "SELECT max(biblionumber) FROM biblio;";
50
my $sthlastbiblionumber   = $dbh->prepare($querylastbiblionumber);
51
$sthlastbiblionumber->execute();
52
my ( $lastbiblionumber ) = $sthlastbiblionumber->fetchrow_array; print $lastbiblionumber;'`
53
#echo $lastbiblionumber;
54
let "maxbiblionumber = $lastbiblionumber + 1"
55
if [ $# = 2 ]
56
then
57
    BIBLIOEND=$lastbiblionumber
58
elif [ $# = 1 ]
59
then
60
    BIBLIOSTART=0
61
    BIBLIOEND=$lastbiblionumber
62
elif [ $# = 0 ]
63
then
64
    INCREMENT=10000
65
    BIBLIOSTART=0
66
    BIBLIOEND=$lastbiblionumber
67
fi
68
if [[ $INCREMENT =~ ^10*$ ]]
69
then
70
else
71
    echo "The first argument (INCREMENT) must be 1 or a multiple of 10"
72
    exit 2
73
fi
74
if [[ $BIBLIOSTART =~ ^[0-9]*$ ]]
75
then
76
else
77
    echo "The second argument (BIBLIOSTART) must be an integer"
78
    exit 2
79
fi
80
if [[ $BIBLIOEND =~ ^[0-9]*$ ]]
81
then
82
else
83
    echo "The third argument (BIBLIOEND) must be an integer"
84
    exit 2
85
fi
86
if [ $BIBLIOSTART -lt $BIBLIOEND ]
87
then
88
else
89
    echo "The second argument (BIBLIOSTART) must be lower than the third argument (BIBLIOEND)"
90
    exit 2
91
fi
92
if [ $BIBLIOEND -lt $maxbiblionumber ]
93
then
94
else
95
    echo "end"
96
    exit 1
97
fi
98
    ls `dirname $0`/logs/ >/dev/null 2>&1
99
    if [ $? != 0 ]
100
    then
101
        mkdir `dirname $0`/logs
102
    else
103
        rm `dirname $0`/logs/*
104
    fi
105
    #/home/koha/src/misc/migration_tools/rebuild_zebra.pl -r -b -v -x -nosanitize -length 1 -offset 1
106
    for ((i=$BIBLIOSTART ; i<$BIBLIOEND ; i=i+$INCREMENT))
107
    do
108
        echo "I = " $i "with increment " $INCREMENT
109
        `dirname $0`/rebuild_zebra.pl -b -v -x -nosanitize -d /tmp/rebuild -k -length $INCREMENT -offset $i > `dirname $0`/logs/rebuild$INCREMENT.$i.log 2> `dirname $0`/logs/rebuild$INCREMENT.$i.err
110
        if (($INCREMENT >1 ));
111
        then
112
            if { grep -q "previous transaction" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ;
113
            then
114
                echo "I must split $i (increment $INCREMENT) because previous transaction didn't reach commit"
115
                ((subincrement=$INCREMENT/10))
116
                ((newBIBLIOEND=$i+$INCREMENT))
117
                $0 $subincrement $i $newBIBLIOEND
118
            elif { ! grep -q "Records: $INCREMENT" `dirname $0`/logs/rebuild$INCREMENT.$i.err } ;
119
            then
120
                echo "I must split $i (increment $INCREMENT) because index was uncomplete, less than $INCREMENT records indexed"
121
                ((subincrement=$INCREMENT/10))
122
                ((newBIBLIOEND=$i+$INCREMENT))
123
                $0 $subincrement $i $newBIBLIOEND
124
            fi
125
        fi
126
    done
127
exit 0

Return to bug 7286