View | Details | Raw Unified | Return to bug 9035
Collapse All | Expand All

(-)a/misc/migration_tools/bulkauthimport.pl (-155 lines)
Lines 1-154 Link Here
1
#!/usr/bin/perl
2
# small script that import an iso2709 file into koha 2.0
3
4
use strict;
5
#use warnings; FIXME - Bug 2505
6
BEGIN {
7
    # find Koha's Perl modules
8
    # test carefully before changing this
9
    use FindBin;
10
    eval { require "$FindBin::Bin/kohalib.pl" };
11
}
12
13
# Koha modules used
14
use Unicode::Normalize;
15
use MARC::File::USMARC;
16
use MARC::File::XML;
17
use MARC::Record;
18
use MARC::Batch;
19
use C4::Context;
20
use C4::Charset;
21
use C4::AuthoritiesMarc;
22
use Time::HiRes qw(gettimeofday);
23
use Getopt::Long;
24
use IO::File;
25
26
my ( $input_marc_file, $number) = ('',0);
27
my ($version, $delete, $test_parameter,$char_encoding, $verbose, $format, $commit);
28
$| = 1;
29
GetOptions(
30
    'file:s'    => \$input_marc_file,
31
    'n:i' => \$number,
32
    'h' => \$version,
33
    'd' => \$delete,
34
    't' => \$test_parameter,
35
    'c:s' => \$char_encoding,
36
    'v:s' => \$verbose,
37
    'm:s' => \$format,
38
    'commit:f' => \$commit,
39
);
40
41
if ($version || ($input_marc_file eq '')) {
42
    print <<EOF
43
small script to import an iso2709 file into Koha.
44
parameters :
45
\th : this version/help screen
46
\tfile /path/to/file/to/dump : the file to dump
47
\tv : verbose mode. Valid modes are 1 and 2
48
\tn : the number of the record to import. If missing, the whole file is imported
49
\tt : test mode : parses the file, saying what it would do, but doing nothing.
50
\tc : the MARC flavour. At the moment, MARC21 and UNIMARC supported. MARC21 by default.
51
\td : delete EVERYTHING related to authorities in koha-DB before import
52
\tm : format, MARCXML or ISO2709
53
\tcommit : the number of records to wait before performing a 'commit' operation
54
IMPORTANT : don't use this script before you've entered and checked twice (or more) your MARC parameters tables.
55
If you fail the test, the import won't work correctly and you will get invalid datas.
56
57
SAMPLE : ./bulkmarcimport.pl -file /home/paul/koha.dev/local/npl -n 1
58
EOF
59
;#'
60
die;
61
}
62
63
my $dbh = C4::Context->dbh;
64
65
if ($delete) {
66
    print "deleting authorities\n";
67
    $dbh->do("truncate auth_header");
68
}
69
if ($test_parameter) {
70
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
71
}
72
73
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
74
$char_encoding = 'MARC21' unless ($char_encoding);
75
print "CHAR : $char_encoding\n" if $verbose;
76
my $starttime = gettimeofday;
77
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
78
my $batch;
79
if ($format =~ /XML/i) {
80
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
81
    # appears to try to convert incoming XML records from MARC-8
82
    # to UTF-8.  Setting the BinaryEncoding key turns that off
83
    # TODO: see what happens to ISO-8859-1 XML files.
84
    # TODO: determine if MARC::Batch can be fixed to handle
85
    #       XML records properly -- it probably should be
86
    #       be using a proper push or pull XML parser to
87
    #       extract the records, not using regexes to look
88
    #       for <record>.*</record>.
89
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
90
    $batch = MARC::Batch->new( 'XML', $fh );
91
} else {
92
    $batch = MARC::Batch->new( 'USMARC', $fh );
93
}
94
$batch->warnings_off();
95
$batch->strict_off();
96
my $i=0;
97
98
$dbh->{AutoCommit} = 0;
99
my $commitnum = 50;
100
if ($commit) {
101
    $commitnum = $commit;
102
}
103
104
RECORD: while ( my $record = $batch->next() ) {
105
    $i++;
106
    print ".";
107
    print "\r$i" unless $i % 100;
108
109
    if ($record->encoding() eq 'MARC-8') {
110
        my ($guessed_charset, $charset_errors);
111
        ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour);
112
        if ($guessed_charset eq 'failed') {
113
            warn "ERROR: failed to perform character conversion for record $i\n";
114
            next RECORD;            
115
        }
116
    }
117
118
    warn "$i ==>".$record->as_formatted() if $verbose eq 2;
119
    my $authtypecode;
120
    if (C4::Context->preference('marcflavour') eq 'MARC21') {
121
        $authtypecode="PERSO_NAME" if ($record->field('100'));
122
        $authtypecode="CORPO_NAME" if ($record->field('110'));
123
        $authtypecode="MEETI_NAME" if ($record->field('111'));
124
        $authtypecode="UNIF_TITLE" if ($record->field('130'));
125
        $authtypecode="CHRON_TERM" if ($record->field('148') or $record->field('182'));
126
        $authtypecode="TOPIC_TERM" if ($record->field('150') or $record->field('180'));
127
        $authtypecode="GEOGR_NAME" if ($record->field('151') or $record->field('181'));
128
        $authtypecode="GENRE/FORM" if ($record->field('155') or $record->field('185'));
129
        next unless $authtypecode; # skip invalid records FIXME: far too simplistic
130
    }
131
    else {
132
        $authtypecode=substr($record->leader(),9,1);
133
        $authtypecode="NP" if ($authtypecode eq 'a'); # personnes
134
        $authtypecode="CO" if ($authtypecode eq 'b'); # collectivit�
135
        $authtypecode="NG" if ($authtypecode eq 'c'); # g�graphique
136
        $authtypecode="NM" if ($authtypecode eq 'd'); # marque
137
        $authtypecode="NF" if ($authtypecode eq 'e'); # famille
138
        $authtypecode="TI" if ($authtypecode eq 'f'); # Titre uniforme
139
        $authtypecode="TI" if ($authtypecode eq 'h'); # auteur/titre
140
        $authtypecode="MM" if ($authtypecode eq 'j'); # mot mati�e
141
    }
142
143
    unless ($test_parameter) {
144
        my ($authid) = AddAuthority($record,0,$authtypecode);
145
        warn "ADDED authority NB $authid in DB\n" if $verbose;
146
        $dbh->commit() if (0 == $i % $commitnum);
147
    }
148
149
    last if $i ==  $number;
150
}
151
$dbh->commit();
152
153
my $timeneeded = gettimeofday - $starttime;
154
print "\n$i MARC record done in $timeneeded seconds\n";
155
- 

Return to bug 9035