|
Lines 1-154
Link Here
|
| 1 |
#!/usr/bin/perl |
|
|
| 2 |
# small script that import an iso2709 file into koha 2.0 |
| 3 |
|
| 4 |
use strict; |
| 5 |
#use warnings; FIXME - Bug 2505 |
| 6 |
BEGIN { |
| 7 |
# find Koha's Perl modules |
| 8 |
# test carefully before changing this |
| 9 |
use FindBin; |
| 10 |
eval { require "$FindBin::Bin/kohalib.pl" }; |
| 11 |
} |
| 12 |
|
| 13 |
# Koha modules used |
| 14 |
use Unicode::Normalize; |
| 15 |
use MARC::File::USMARC; |
| 16 |
use MARC::File::XML; |
| 17 |
use MARC::Record; |
| 18 |
use MARC::Batch; |
| 19 |
use C4::Context; |
| 20 |
use C4::Charset; |
| 21 |
use C4::AuthoritiesMarc; |
| 22 |
use Time::HiRes qw(gettimeofday); |
| 23 |
use Getopt::Long; |
| 24 |
use IO::File; |
| 25 |
|
| 26 |
my ( $input_marc_file, $number) = ('',0); |
| 27 |
my ($version, $delete, $test_parameter,$char_encoding, $verbose, $format, $commit); |
| 28 |
$| = 1; |
| 29 |
GetOptions( |
| 30 |
'file:s' => \$input_marc_file, |
| 31 |
'n:i' => \$number, |
| 32 |
'h' => \$version, |
| 33 |
'd' => \$delete, |
| 34 |
't' => \$test_parameter, |
| 35 |
'c:s' => \$char_encoding, |
| 36 |
'v:s' => \$verbose, |
| 37 |
'm:s' => \$format, |
| 38 |
'commit:f' => \$commit, |
| 39 |
); |
| 40 |
|
| 41 |
if ($version || ($input_marc_file eq '')) { |
| 42 |
print <<EOF |
| 43 |
small script to import an iso2709 file into Koha. |
| 44 |
parameters : |
| 45 |
\th : this version/help screen |
| 46 |
\tfile /path/to/file/to/dump : the file to dump |
| 47 |
\tv : verbose mode. Valid modes are 1 and 2 |
| 48 |
\tn : the number of the record to import. If missing, the whole file is imported |
| 49 |
\tt : test mode : parses the file, saying what it would do, but doing nothing. |
| 50 |
\tc : the MARC flavour. At the moment, MARC21 and UNIMARC supported. MARC21 by default. |
| 51 |
\td : delete EVERYTHING related to authorities in koha-DB before import |
| 52 |
\tm : format, MARCXML or ISO2709 |
| 53 |
\tcommit : the number of records to wait before performing a 'commit' operation |
| 54 |
IMPORTANT : don't use this script before you've entered and checked twice (or more) your MARC parameters tables. |
| 55 |
If you fail the test, the import won't work correctly and you will get invalid datas. |
| 56 |
|
| 57 |
SAMPLE : ./bulkmarcimport.pl -file /home/paul/koha.dev/local/npl -n 1 |
| 58 |
EOF |
| 59 |
;#' |
| 60 |
die; |
| 61 |
} |
| 62 |
|
| 63 |
my $dbh = C4::Context->dbh; |
| 64 |
|
| 65 |
if ($delete) { |
| 66 |
print "deleting authorities\n"; |
| 67 |
$dbh->do("truncate auth_header"); |
| 68 |
} |
| 69 |
if ($test_parameter) { |
| 70 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
| 71 |
} |
| 72 |
|
| 73 |
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21'; |
| 74 |
$char_encoding = 'MARC21' unless ($char_encoding); |
| 75 |
print "CHAR : $char_encoding\n" if $verbose; |
| 76 |
my $starttime = gettimeofday; |
| 77 |
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
| 78 |
my $batch; |
| 79 |
if ($format =~ /XML/i) { |
| 80 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
| 81 |
# appears to try to convert incoming XML records from MARC-8 |
| 82 |
# to UTF-8. Setting the BinaryEncoding key turns that off |
| 83 |
# TODO: see what happens to ISO-8859-1 XML files. |
| 84 |
# TODO: determine if MARC::Batch can be fixed to handle |
| 85 |
# XML records properly -- it probably should be |
| 86 |
# be using a proper push or pull XML parser to |
| 87 |
# extract the records, not using regexes to look |
| 88 |
# for <record>.*</record>. |
| 89 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
| 90 |
$batch = MARC::Batch->new( 'XML', $fh ); |
| 91 |
} else { |
| 92 |
$batch = MARC::Batch->new( 'USMARC', $fh ); |
| 93 |
} |
| 94 |
$batch->warnings_off(); |
| 95 |
$batch->strict_off(); |
| 96 |
my $i=0; |
| 97 |
|
| 98 |
$dbh->{AutoCommit} = 0; |
| 99 |
my $commitnum = 50; |
| 100 |
if ($commit) { |
| 101 |
$commitnum = $commit; |
| 102 |
} |
| 103 |
|
| 104 |
RECORD: while ( my $record = $batch->next() ) { |
| 105 |
$i++; |
| 106 |
print "."; |
| 107 |
print "\r$i" unless $i % 100; |
| 108 |
|
| 109 |
if ($record->encoding() eq 'MARC-8') { |
| 110 |
my ($guessed_charset, $charset_errors); |
| 111 |
($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour); |
| 112 |
if ($guessed_charset eq 'failed') { |
| 113 |
warn "ERROR: failed to perform character conversion for record $i\n"; |
| 114 |
next RECORD; |
| 115 |
} |
| 116 |
} |
| 117 |
|
| 118 |
warn "$i ==>".$record->as_formatted() if $verbose eq 2; |
| 119 |
my $authtypecode; |
| 120 |
if (C4::Context->preference('marcflavour') eq 'MARC21') { |
| 121 |
$authtypecode="PERSO_NAME" if ($record->field('100')); |
| 122 |
$authtypecode="CORPO_NAME" if ($record->field('110')); |
| 123 |
$authtypecode="MEETI_NAME" if ($record->field('111')); |
| 124 |
$authtypecode="UNIF_TITLE" if ($record->field('130')); |
| 125 |
$authtypecode="CHRON_TERM" if ($record->field('148') or $record->field('182')); |
| 126 |
$authtypecode="TOPIC_TERM" if ($record->field('150') or $record->field('180')); |
| 127 |
$authtypecode="GEOGR_NAME" if ($record->field('151') or $record->field('181')); |
| 128 |
$authtypecode="GENRE/FORM" if ($record->field('155') or $record->field('185')); |
| 129 |
next unless $authtypecode; # skip invalid records FIXME: far too simplistic |
| 130 |
} |
| 131 |
else { |
| 132 |
$authtypecode=substr($record->leader(),9,1); |
| 133 |
$authtypecode="NP" if ($authtypecode eq 'a'); # personnes |
| 134 |
$authtypecode="CO" if ($authtypecode eq 'b'); # collectivit� |
| 135 |
$authtypecode="NG" if ($authtypecode eq 'c'); # g�graphique |
| 136 |
$authtypecode="NM" if ($authtypecode eq 'd'); # marque |
| 137 |
$authtypecode="NF" if ($authtypecode eq 'e'); # famille |
| 138 |
$authtypecode="TI" if ($authtypecode eq 'f'); # Titre uniforme |
| 139 |
$authtypecode="TI" if ($authtypecode eq 'h'); # auteur/titre |
| 140 |
$authtypecode="MM" if ($authtypecode eq 'j'); # mot mati�e |
| 141 |
} |
| 142 |
|
| 143 |
unless ($test_parameter) { |
| 144 |
my ($authid) = AddAuthority($record,0,$authtypecode); |
| 145 |
warn "ADDED authority NB $authid in DB\n" if $verbose; |
| 146 |
$dbh->commit() if (0 == $i % $commitnum); |
| 147 |
} |
| 148 |
|
| 149 |
last if $i == $number; |
| 150 |
} |
| 151 |
$dbh->commit(); |
| 152 |
|
| 153 |
my $timeneeded = gettimeofday - $starttime; |
| 154 |
print "\n$i MARC record done in $timeneeded seconds\n"; |
| 155 |
- |