Lines 1-154
Link Here
|
1 |
#!/usr/bin/perl |
|
|
2 |
# small script that import an iso2709 file into koha 2.0 |
3 |
|
4 |
use strict; |
5 |
#use warnings; FIXME - Bug 2505 |
6 |
BEGIN { |
7 |
# find Koha's Perl modules |
8 |
# test carefully before changing this |
9 |
use FindBin; |
10 |
eval { require "$FindBin::Bin/kohalib.pl" }; |
11 |
} |
12 |
|
13 |
# Koha modules used |
14 |
use Unicode::Normalize; |
15 |
use MARC::File::USMARC; |
16 |
use MARC::File::XML; |
17 |
use MARC::Record; |
18 |
use MARC::Batch; |
19 |
use C4::Context; |
20 |
use C4::Charset; |
21 |
use C4::AuthoritiesMarc; |
22 |
use Time::HiRes qw(gettimeofday); |
23 |
use Getopt::Long; |
24 |
use IO::File; |
25 |
|
26 |
my ( $input_marc_file, $number) = ('',0); |
27 |
my ($version, $delete, $test_parameter,$char_encoding, $verbose, $format, $commit); |
28 |
$| = 1; |
29 |
GetOptions( |
30 |
'file:s' => \$input_marc_file, |
31 |
'n:i' => \$number, |
32 |
'h' => \$version, |
33 |
'd' => \$delete, |
34 |
't' => \$test_parameter, |
35 |
'c:s' => \$char_encoding, |
36 |
'v:s' => \$verbose, |
37 |
'm:s' => \$format, |
38 |
'commit:f' => \$commit, |
39 |
); |
40 |
|
41 |
if ($version || ($input_marc_file eq '')) { |
42 |
print <<EOF |
43 |
small script to import an iso2709 file into Koha. |
44 |
parameters : |
45 |
\th : this version/help screen |
46 |
\tfile /path/to/file/to/dump : the file to dump |
47 |
\tv : verbose mode. Valid modes are 1 and 2 |
48 |
\tn : the number of the record to import. If missing, the whole file is imported |
49 |
\tt : test mode : parses the file, saying what it would do, but doing nothing. |
50 |
\tc : the MARC flavour. At the moment, MARC21 and UNIMARC supported. MARC21 by default. |
51 |
\td : delete EVERYTHING related to authorities in koha-DB before import |
52 |
\tm : format, MARCXML or ISO2709 |
53 |
\tcommit : the number of records to wait before performing a 'commit' operation |
54 |
IMPORTANT : don't use this script before you've entered and checked twice (or more) your MARC parameters tables. |
55 |
If you fail the test, the import won't work correctly and you will get invalid datas. |
56 |
|
57 |
SAMPLE : ./bulkmarcimport.pl -file /home/paul/koha.dev/local/npl -n 1 |
58 |
EOF |
59 |
;#' |
60 |
die; |
61 |
} |
62 |
|
63 |
my $dbh = C4::Context->dbh; |
64 |
|
65 |
if ($delete) { |
66 |
print "deleting authorities\n"; |
67 |
$dbh->do("truncate auth_header"); |
68 |
} |
69 |
if ($test_parameter) { |
70 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
71 |
} |
72 |
|
73 |
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21'; |
74 |
$char_encoding = 'MARC21' unless ($char_encoding); |
75 |
print "CHAR : $char_encoding\n" if $verbose; |
76 |
my $starttime = gettimeofday; |
77 |
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
78 |
my $batch; |
79 |
if ($format =~ /XML/i) { |
80 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
81 |
# appears to try to convert incoming XML records from MARC-8 |
82 |
# to UTF-8. Setting the BinaryEncoding key turns that off |
83 |
# TODO: see what happens to ISO-8859-1 XML files. |
84 |
# TODO: determine if MARC::Batch can be fixed to handle |
85 |
# XML records properly -- it probably should be |
86 |
# be using a proper push or pull XML parser to |
87 |
# extract the records, not using regexes to look |
88 |
# for <record>.*</record>. |
89 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
90 |
$batch = MARC::Batch->new( 'XML', $fh ); |
91 |
} else { |
92 |
$batch = MARC::Batch->new( 'USMARC', $fh ); |
93 |
} |
94 |
$batch->warnings_off(); |
95 |
$batch->strict_off(); |
96 |
my $i=0; |
97 |
|
98 |
$dbh->{AutoCommit} = 0; |
99 |
my $commitnum = 50; |
100 |
if ($commit) { |
101 |
$commitnum = $commit; |
102 |
} |
103 |
|
104 |
RECORD: while ( my $record = $batch->next() ) { |
105 |
$i++; |
106 |
print "."; |
107 |
print "\r$i" unless $i % 100; |
108 |
|
109 |
if ($record->encoding() eq 'MARC-8') { |
110 |
my ($guessed_charset, $charset_errors); |
111 |
($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour); |
112 |
if ($guessed_charset eq 'failed') { |
113 |
warn "ERROR: failed to perform character conversion for record $i\n"; |
114 |
next RECORD; |
115 |
} |
116 |
} |
117 |
|
118 |
warn "$i ==>".$record->as_formatted() if $verbose eq 2; |
119 |
my $authtypecode; |
120 |
if (C4::Context->preference('marcflavour') eq 'MARC21') { |
121 |
$authtypecode="PERSO_NAME" if ($record->field('100')); |
122 |
$authtypecode="CORPO_NAME" if ($record->field('110')); |
123 |
$authtypecode="MEETI_NAME" if ($record->field('111')); |
124 |
$authtypecode="UNIF_TITLE" if ($record->field('130')); |
125 |
$authtypecode="CHRON_TERM" if ($record->field('148') or $record->field('182')); |
126 |
$authtypecode="TOPIC_TERM" if ($record->field('150') or $record->field('180')); |
127 |
$authtypecode="GEOGR_NAME" if ($record->field('151') or $record->field('181')); |
128 |
$authtypecode="GENRE/FORM" if ($record->field('155') or $record->field('185')); |
129 |
next unless $authtypecode; # skip invalid records FIXME: far too simplistic |
130 |
} |
131 |
else { |
132 |
$authtypecode=substr($record->leader(),9,1); |
133 |
$authtypecode="NP" if ($authtypecode eq 'a'); # personnes |
134 |
$authtypecode="CO" if ($authtypecode eq 'b'); # collectivit� |
135 |
$authtypecode="NG" if ($authtypecode eq 'c'); # g�graphique |
136 |
$authtypecode="NM" if ($authtypecode eq 'd'); # marque |
137 |
$authtypecode="NF" if ($authtypecode eq 'e'); # famille |
138 |
$authtypecode="TI" if ($authtypecode eq 'f'); # Titre uniforme |
139 |
$authtypecode="TI" if ($authtypecode eq 'h'); # auteur/titre |
140 |
$authtypecode="MM" if ($authtypecode eq 'j'); # mot mati�e |
141 |
} |
142 |
|
143 |
unless ($test_parameter) { |
144 |
my ($authid) = AddAuthority($record,0,$authtypecode); |
145 |
warn "ADDED authority NB $authid in DB\n" if $verbose; |
146 |
$dbh->commit() if (0 == $i % $commitnum); |
147 |
} |
148 |
|
149 |
last if $i == $number; |
150 |
} |
151 |
$dbh->commit(); |
152 |
|
153 |
my $timeneeded = gettimeofday - $starttime; |
154 |
print "\n$i MARC record done in $timeneeded seconds\n"; |
155 |
- |