Lines 1-11
Link Here
|
1 |
#!/usr/bin/perl |
1 |
#!/usr/bin/perl |
2 |
# Import an iso2709 file into Koha 3 |
2 |
# Import an iso2709 or MARCXML file into Koha |
3 |
|
3 |
|
4 |
use Modern::Perl; |
4 |
use Modern::Perl; |
5 |
|
5 |
|
6 |
#use diagnostics; |
|
|
7 |
|
8 |
# Koha modules used |
6 |
# Koha modules used |
|
|
7 |
use MARC::Record; |
9 |
use MARC::File::USMARC; |
8 |
use MARC::File::USMARC; |
10 |
use MARC::File::XML; |
9 |
use MARC::File::XML; |
11 |
use MARC::Batch; |
10 |
use MARC::Batch; |
Lines 29-35
use C4::MarcModificationTemplates qw(
Link Here
|
29 |
GetModificationTemplates |
28 |
GetModificationTemplates |
30 |
ModifyRecordWithTemplate |
29 |
ModifyRecordWithTemplate |
31 |
); |
30 |
); |
32 |
use C4::AuthoritiesMarc qw( GuessAuthTypeCode GuessAuthId GetAuthority ModAuthority AddAuthority ); |
31 |
use C4::AuthoritiesMarc qw( |
|
|
32 |
GuessAuthTypeCode |
33 |
GuessAuthId |
34 |
GetAuthority |
35 |
ModAuthority |
36 |
AddAuthority |
37 |
); |
33 |
|
38 |
|
34 |
use YAML::XS; |
39 |
use YAML::XS; |
35 |
use Time::HiRes qw( gettimeofday ); |
40 |
use Time::HiRes qw( gettimeofday ); |
Lines 45-50
use Koha::SearchEngine::Search;
Link Here
|
45 |
|
50 |
|
46 |
use open qw( :std :encoding(UTF-8) ); |
51 |
use open qw( :std :encoding(UTF-8) ); |
47 |
binmode( STDOUT, ":encoding(UTF-8)" ); |
52 |
binmode( STDOUT, ":encoding(UTF-8)" ); |
|
|
53 |
|
54 |
### CHANGED: We add XML::Twig for streaming MARCXML |
55 |
use XML::Twig; |
56 |
|
48 |
my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 ); |
57 |
my ( $input_marc_file, $number, $offset, $cleanisbn ) = ( '', 0, 0, 1 ); |
49 |
my $version; |
58 |
my $version; |
50 |
my $delete; |
59 |
my $delete; |
Lines 66-72
my $filters;
Link Here
|
66 |
my $update; |
75 |
my $update; |
67 |
my $all; |
76 |
my $all; |
68 |
my $yamlfile; |
77 |
my $yamlfile; |
69 |
my $authtypes; |
|
|
70 |
my $append; |
78 |
my $append; |
71 |
my $sourcetag; |
79 |
my $sourcetag; |
72 |
my $sourcesubfield; |
80 |
my $sourcesubfield; |
Lines 78-83
my $marc_mod_template = '';
Link Here
|
78 |
my $marc_mod_template_id = -1; |
86 |
my $marc_mod_template_id = -1; |
79 |
my $skip_indexing = 0; |
87 |
my $skip_indexing = 0; |
80 |
my $skip_bad_records; |
88 |
my $skip_bad_records; |
|
|
89 |
|
81 |
$| = 1; |
90 |
$| = 1; |
82 |
|
91 |
|
83 |
GetOptions( |
92 |
GetOptions( |
Lines 120-129
GetOptions(
Link Here
|
120 |
|
129 |
|
121 |
$biblios ||= !$authorities; |
130 |
$biblios ||= !$authorities; |
122 |
$insert ||= !$update; |
131 |
$insert ||= !$update; |
123 |
my $writemode = ($append) ? "a" : "w"; |
132 |
my $writemode = ( $append ) ? "a" : "w"; |
124 |
|
133 |
|
125 |
pod2usage( -msg => "\nYou must specify either --biblios or --authorities, not both.\n", -exitval ) |
134 |
pod2usage( |
126 |
if $biblios && $authorities; |
135 |
-msg => "\nYou must specify either --biblios or --authorities, not both.\n", |
|
|
136 |
-exitval => 1 |
137 |
) if $biblios && $authorities; |
127 |
|
138 |
|
128 |
if ($all) { |
139 |
if ($all) { |
129 |
$insert = 1; |
140 |
$insert = 1; |
Lines 134-147
my $using_elastic_search = ( C4::Context->preference('SearchEngine') eq 'Elastic
Link Here
|
134 |
my $mod_biblio_options = { |
145 |
my $mod_biblio_options = { |
135 |
disable_autolink => $using_elastic_search, |
146 |
disable_autolink => $using_elastic_search, |
136 |
skip_record_index => $using_elastic_search || $skip_indexing, |
147 |
skip_record_index => $using_elastic_search || $skip_indexing, |
137 |
overlay_context => { source => 'bulkmarcimport' } |
148 |
overlay_context => { source => 'bulkmarcimport' }, |
138 |
}; |
149 |
}; |
139 |
my $add_biblio_options = { |
150 |
my $add_biblio_options = { |
140 |
disable_autolink => $using_elastic_search, |
151 |
disable_autolink => $using_elastic_search, |
|
|
152 |
skip_record_index => $using_elastic_search || $skip_indexing, |
153 |
}; |
154 |
my $mod_authority_options = { |
155 |
skip_record_index => $using_elastic_search || $skip_indexing |
156 |
}; |
157 |
my $add_authority_options = { |
141 |
skip_record_index => $using_elastic_search || $skip_indexing |
158 |
skip_record_index => $using_elastic_search || $skip_indexing |
142 |
}; |
159 |
}; |
143 |
my $mod_authority_options = { skip_record_index => $using_elastic_search || $skip_indexing }; |
|
|
144 |
my $add_authority_options = { skip_record_index => $using_elastic_search || $skip_indexing }; |
145 |
|
160 |
|
146 |
my @search_engine_record_ids; |
161 |
my @search_engine_record_ids; |
147 |
my @search_engine_records; |
162 |
my @search_engine_records; |
Lines 151-176
if ($using_elastic_search) {
Link Here
|
151 |
$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( |
166 |
$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( |
152 |
{ |
167 |
{ |
153 |
index => $authorities |
168 |
index => $authorities |
154 |
? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX |
169 |
? $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX |
155 |
: $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX |
170 |
: $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX |
156 |
} |
171 |
} |
157 |
); |
172 |
); |
158 |
} |
173 |
} |
159 |
|
174 |
|
160 |
if ( $version || ( $input_marc_file eq '' ) ) { |
175 |
if ( $version || $input_marc_file eq '' ) { |
161 |
pod2usage( -verbose => 2 ); |
176 |
pod2usage( -verbose => 2 ); |
162 |
exit; |
177 |
exit; |
163 |
} |
178 |
} |
164 |
if ( $update && !( $match || $isbn_check ) ) { |
179 |
if ( $update && !( $match || $isbn_check ) ) { |
165 |
warn "Using --update without --match or --isbn seems to be useless.\n"; |
180 |
warn "Using --update without --match or --isbn is likely not useful.\n"; |
166 |
} |
181 |
} |
167 |
|
182 |
|
168 |
if ( defined $localcust ) { #local customize module |
183 |
if ( defined $localcust ) { |
169 |
if ( !-e $localcust ) { |
184 |
if ( !-e $localcust ) { |
170 |
$localcust = $localcust || 'LocalChanges'; #default name |
185 |
$localcust = $localcust || 'LocalChanges'; |
171 |
$localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only |
186 |
$localcust =~ s/^.*\/([^\/]+)$/$1/; |
172 |
$localcust =~ s/\.pm$//; #remove extension |
187 |
$localcust =~ s/\.pm$//; |
173 |
my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir |
188 |
my $fqcust = $FindBin::Bin . "/$localcust.pm"; |
174 |
if ( -e $fqcust ) { |
189 |
if ( -e $fqcust ) { |
175 |
$localcust = $fqcust; |
190 |
$localcust = $fqcust; |
176 |
} else { |
191 |
} else { |
Lines 190-212
if ( $marc_mod_template ne '' ) {
Link Here
|
190 |
$marc_mod_template_id = $this_template->{'template_id'}; |
205 |
$marc_mod_template_id = $this_template->{'template_id'}; |
191 |
} else { |
206 |
} else { |
192 |
print "WARNING: MARC modification template name " |
207 |
print "WARNING: MARC modification template name " |
193 |
. "'$marc_mod_template' matches multiple templates. " |
208 |
. "'$marc_mod_template' matches multiple templates.\n"; |
194 |
. "Please rename these templates\n"; |
|
|
195 |
exit 1; |
209 |
exit 1; |
196 |
} |
210 |
} |
197 |
} |
211 |
} |
198 |
} |
212 |
} |
199 |
if ( $marc_mod_template_id < 0 ) { |
213 |
if ( $marc_mod_template_id < 0 ) { |
200 |
die "Can't located MARC modification template '$marc_mod_template'\n"; |
214 |
die "Can't locate MARC modification template '$marc_mod_template'\n"; |
201 |
} else { |
215 |
} else { |
202 |
print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose; |
216 |
print "MARC modification template: $marc_mod_template\n" if $verbose; |
203 |
} |
217 |
} |
204 |
} |
218 |
} |
205 |
|
219 |
|
206 |
my $dbh = C4::Context->dbh; |
220 |
my $dbh = C4::Context->dbh; |
207 |
my $heading_fields = get_heading_fields(); |
221 |
my $heading_fields = get_heading_fields($authtypes); |
208 |
my $idmapfh; |
222 |
my $idmapfh; |
209 |
|
|
|
210 |
if ( defined $idmapfl ) { |
223 |
if ( defined $idmapfl ) { |
211 |
open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n"; |
224 |
open( $idmapfh, '>', $idmapfl ) or die "cannot open $idmapfl \n"; |
212 |
} |
225 |
} |
Lines 219-224
if ( ( not defined $sourcesubfield ) && ( not defined $sourcetag ) ) {
Link Here
|
219 |
# Disable logging for the biblios and authorities import operation. It would unnecessarily |
232 |
# Disable logging for the biblios and authorities import operation. It would unnecessarily |
220 |
# slow the import |
233 |
# slow the import |
221 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
234 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
|
|
235 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
222 |
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0; |
236 |
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0; |
223 |
|
237 |
|
224 |
if ($fk_off) { |
238 |
if ($fk_off) { |
Lines 236-318
if ($delete) {
Link Here
|
236 |
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); |
250 |
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); |
237 |
} else { |
251 |
} else { |
238 |
print "Deleting authorities\n"; |
252 |
print "Deleting authorities\n"; |
239 |
$dbh->do("truncate auth_header"); |
253 |
$dbh->do("TRUNCATE auth_header"); |
240 |
} |
254 |
} |
241 |
$dbh->do("truncate zebraqueue"); |
255 |
$dbh->do("TRUNCATE zebraqueue"); |
242 |
} |
256 |
} |
243 |
|
257 |
|
244 |
if ($test_parameter) { |
258 |
if ($test_parameter) { |
245 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
259 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
246 |
} |
260 |
} |
247 |
|
261 |
|
248 |
my $batch; |
|
|
249 |
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21'; |
262 |
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21'; |
|
|
263 |
print "MARC flavour: $marc_flavour\n" if $verbose; |
250 |
|
264 |
|
251 |
# The definition of $searcher must be before MARC::Batch->new |
265 |
# The definition of $searcher must be before MARC::Batch->new |
252 |
my $searcher = Koha::SearchEngine::Search->new( |
266 |
my $searcher = Koha::SearchEngine::Search->new( |
253 |
{ |
267 |
{ |
254 |
index => ( |
268 |
index => $authorities |
255 |
$authorities |
|
|
256 |
? $Koha::SearchEngine::AUTHORITIES_INDEX |
269 |
? $Koha::SearchEngine::AUTHORITIES_INDEX |
257 |
: $Koha::SearchEngine::BIBLIOS_INDEX |
270 |
: $Koha::SearchEngine::BIBLIOS_INDEX |
258 |
) |
|
|
259 |
} |
271 |
} |
260 |
); |
272 |
); |
261 |
|
273 |
|
262 |
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose; |
274 |
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose; |
263 |
my $starttime = gettimeofday; |
275 |
my $starttime = gettimeofday; |
264 |
|
276 |
my $logger = Koha::Logger->get; |
265 |
# don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
277 |
my $schema = Koha::Database->schema; |
266 |
my $fh = IO::File->new($input_marc_file) or die "Could not open $input_marc_file: $!"; |
278 |
my $lint = MARC::Lint->new; |
267 |
|
|
|
268 |
if ( defined $format && $format =~ /XML/i ) { |
269 |
|
270 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
271 |
# appears to try to convert incoming XML records from MARC-8 |
272 |
# to UTF-8. Setting the BinaryEncoding key turns that off |
273 |
# TODO: see what happens to ISO-8859-1 XML files. |
274 |
# TODO: determine if MARC::Batch can be fixed to handle |
275 |
# XML records properly -- it probably should be |
276 |
# be using a proper push or pull XML parser to |
277 |
# extract the records, not using regexes to look |
278 |
# for <record>.*</record>. |
279 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
280 |
my $recordformat = ( $marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour) ); |
281 |
|
282 |
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. |
283 |
$recordformat = $recordformat . "AUTH" if ( $authorities and $marc_flavour ne "MARC21" ); |
284 |
$MARC::File::XML::_load_args{RecordFormat} = $recordformat; |
285 |
$batch = MARC::Batch->new( 'XML', $fh ); |
286 |
} else { |
287 |
$batch = MARC::Batch->new( 'USMARC', $fh ); |
288 |
} |
289 |
|
290 |
$batch->warnings_off(); |
291 |
$batch->strict_off(); |
292 |
my $commitnum = $commit ? $commit : 50; |
293 |
my $yamlhash; |
294 |
|
295 |
# Skip file offset |
296 |
if ($offset) { |
297 |
print "Skipping file offset: $offset records\n"; |
298 |
$batch->next() while ( $offset-- ); |
299 |
} |
300 |
|
301 |
my ( $tagid, $subfieldid ); |
302 |
if ($authorities) { |
303 |
$tagid = '001'; |
304 |
} else { |
305 |
( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber"); |
306 |
$tagid ||= "001"; |
307 |
} |
308 |
|
309 |
my $sth_isbn; |
310 |
|
311 |
# the SQL query to search on isbn |
312 |
if ($isbn_check) { |
313 |
$sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?"); |
314 |
} |
315 |
|
316 |
my $loghandle; |
279 |
my $loghandle; |
317 |
if ($logfile) { |
280 |
if ($logfile) { |
318 |
$loghandle = IO::File->new( $logfile, $writemode ); |
281 |
$loghandle = IO::File->new( $logfile, $writemode ); |
Lines 320-399
if ($logfile) {
Link Here
|
320 |
} |
283 |
} |
321 |
|
284 |
|
322 |
my $record_number = 0; |
285 |
my $record_number = 0; |
323 |
my $logger = Koha::Logger->get; |
286 |
my $commitnum = $commit ? $commit : 50; |
324 |
my $schema = Koha::Database->schema; |
287 |
my $yamlhash; |
325 |
my $marc_records = []; |
|
|
326 |
my $lint = MARC::Lint->new; |
327 |
RECORD: while () { |
328 |
my $record; |
329 |
$record_number++; |
330 |
|
288 |
|
331 |
# get record |
289 |
$schema->txn_begin; |
332 |
eval { $record = $batch->next() }; |
|
|
333 |
if ($@) { |
334 |
print "Bad MARC record $record_number: $@ skipped\n"; |
335 |
|
336 |
# FIXME - because MARC::Batch->next() combines grabbing the next |
337 |
# blob and parsing it into one operation, a correctable condition |
338 |
# such as a MARC-8 record claiming that it's UTF-8 can't be recovered |
339 |
# from because we don't have access to the original blob. Note |
340 |
# that the staging import can deal with this condition (via |
341 |
# C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. |
342 |
next; |
343 |
} |
344 |
if ($record) { |
345 |
|
290 |
|
346 |
if ($skip_bad_records) { |
291 |
### CHANGED: A new sub to handle "one MARC::Record" import logic. |
347 |
my $xml = $record->as_xml_record(); |
292 |
### We take the logic from the old while RECORD loop, but wrap in a sub. |
348 |
eval { MARC::Record::new_from_xml( $xml, 'UTF-8', "MARC21" ); }; |
293 |
sub import_single_record { |
349 |
if ($@) { |
294 |
my ($record) = @_; |
350 |
print "Record $record_number generated invalid xml:\n"; |
295 |
return unless $record; |
351 |
$lint->check_record($record); |
296 |
|
352 |
foreach my $warning ( $lint->warnings ) { |
297 |
# increment once per record |
353 |
print " " . $warning . "\n"; |
298 |
$record_number++; |
354 |
} |
299 |
|
355 |
print " Record skipped!"; |
300 |
# skip Bad records if requested |
356 |
next; |
301 |
if ($skip_bad_records) { |
|
|
302 |
my $xml = $record->as_xml_record(); |
303 |
eval { MARC::Record::new_from_xml( $xml, 'UTF-8', "MARC21" ); }; |
304 |
if ($@) { |
305 |
print "Record $record_number is invalid, skipping.\n"; |
306 |
$lint->check_record($record); |
307 |
foreach my $warning ( $lint->warnings ) { |
308 |
print " Lint: $warning\n"; |
357 |
} |
309 |
} |
|
|
310 |
return; |
358 |
} |
311 |
} |
359 |
# transcode the record to UTF8 if needed & applicable. |
312 |
} |
360 |
if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) { |
313 |
|
361 |
my ( $guessed_charset, $charset_errors ); |
314 |
# If record is MARC-8 and not skipping |
362 |
( $record, $guessed_charset, $charset_errors ) = MarcToUTF8Record( |
315 |
if ( $record->encoding() eq 'MARC-8' and not $skip_marc8_conversion ) { |
363 |
$record, |
316 |
my ( $newrec, $charset_guess, $charset_err ) = MarcToUTF8Record( |
364 |
$marc_flavour . ( ( $authorities and $marc_flavour ne "MARC21" ) ? 'AUTH' : '' ) |
317 |
$record, |
365 |
); |
318 |
$marc_flavour . ( ($authorities && $marc_flavour ne 'MARC21') ? 'AUTH' : '' ) |
366 |
if ( $guessed_charset eq 'failed' ) { |
319 |
); |
367 |
warn "ERROR: failed to perform character conversion for record $record_number\n"; |
320 |
if ( $charset_guess eq 'failed' ) { |
368 |
next RECORD; |
321 |
warn "Charset conversion failed for record #$record_number\n"; |
369 |
} |
322 |
return; |
370 |
} |
323 |
} |
371 |
SetUTF8Flag($record); |
324 |
$record = $newrec; |
372 |
&$localcust($record) if $localcust; |
|
|
373 |
push @{$marc_records}, $record; |
374 |
} else { |
375 |
last; |
376 |
} |
325 |
} |
377 |
} |
326 |
SetUTF8Flag($record); |
378 |
|
327 |
|
379 |
$record_number = 0; |
328 |
# local custom routine |
380 |
my $records_total = @{$marc_records}; |
329 |
&$localcust($record) if $localcust; |
381 |
$schema->txn_begin; |
330 |
|
382 |
RECORD: foreach my $record ( @{$marc_records} ) { |
331 |
if ( ( $verbose // 0 ) == 1 ) { |
383 |
$record_number++; |
|
|
384 |
if ( ( $verbose // 1 ) == 1 ) { #no dot for verbose==2 |
385 |
print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' ); |
332 |
print "." . ( $record_number % 100 == 0 ? "\n$record_number" : '' ); |
|
|
333 |
} elsif ( ( $verbose // 0 ) > 1 ) { |
334 |
print $record->as_formatted(), "\n"; |
386 |
} |
335 |
} |
387 |
|
336 |
|
|
|
337 |
# apply MARC mod template |
388 |
if ( $marc_mod_template_id > 0 ) { |
338 |
if ( $marc_mod_template_id > 0 ) { |
389 |
print "Modifying MARC\n" if $verbose; |
|
|
390 |
ModifyRecordWithTemplate( $marc_mod_template_id, $record ); |
339 |
ModifyRecordWithTemplate( $marc_mod_template_id, $record ); |
391 |
} |
340 |
} |
392 |
|
341 |
|
|
|
342 |
# Possibly handle ISBN cleaning |
393 |
my $isbn; |
343 |
my $isbn; |
394 |
|
344 |
if ($biblios && ( $cleanisbn || $isbn_check )) { |
395 |
# remove trailing - in isbn (only for biblios, of course) |
|
|
396 |
if ( $biblios && ( $cleanisbn || $isbn_check ) ) { |
397 |
my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; |
345 |
my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; |
398 |
my $field = $record->field($tag); |
346 |
my $field = $record->field($tag); |
399 |
$isbn = $field && $field->subfield('a'); |
347 |
$isbn = $field && $field->subfield('a'); |
Lines 403-430
RECORD: foreach my $record ( @{$marc_records} ) {
Link Here
|
403 |
} |
351 |
} |
404 |
} |
352 |
} |
405 |
|
353 |
|
406 |
# search for duplicates (based on Local-number) |
354 |
my ( $tagid, $subfieldid ); |
|
|
355 |
if ($authorities) { |
356 |
$tagid = '001'; |
357 |
} else { |
358 |
( $tagid, $subfieldid ) = GetMarcFromKohaField("biblio.biblionumber"); |
359 |
$tagid ||= '001'; |
360 |
} |
361 |
|
407 |
my $originalid = GetRecordId( $record, $tagid, $subfieldid ); |
362 |
my $originalid = GetRecordId( $record, $tagid, $subfieldid ); |
408 |
my $matched_record_id = undef; |
363 |
my $matched_record_id = undef; |
|
|
364 |
|
365 |
# handle match logic |
409 |
if ($match) { |
366 |
if ($match) { |
410 |
require C4::Search; |
367 |
require C4::Search; |
411 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
368 |
my $server = $authorities ? 'authorityserver' : 'biblioserver'; |
412 |
my $query = build_query( $match, $record ); |
369 |
my $query = build_query( $match, $record ); |
413 |
$logger->debug("Bulkmarcimport: $query"); |
370 |
$logger->debug("Bulkmarcimport: $query"); |
414 |
my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] ); |
371 |
my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] ); |
415 |
|
|
|
416 |
# changed to warn so able to continue with one broken record |
417 |
if ( defined $error ) { |
372 |
if ( defined $error ) { |
418 |
warn "unable to search the database for duplicates : $error"; |
373 |
warn "Duplicate search error: $error\n"; |
419 |
printlog( { id => $originalid, op => "match", status => "ERROR" } ) if ($logfile); |
374 |
printlog({ id => $originalid, op => "match", status => "ERROR" }) if $loghandle; |
420 |
next RECORD; |
375 |
return; |
421 |
} |
376 |
} |
422 |
$logger->debug("Bulkmarcimport: $query $server : $totalhits"); |
377 |
$logger->debug("Bulkmarcimport: $query $server : $totalhits"); |
423 |
|
378 |
|
424 |
# sub SimpleSearch could return undefined, but only on error, so |
|
|
425 |
# should not really need to safeguard here, but do so anyway |
426 |
$results //= []; |
379 |
$results //= []; |
427 |
if ( @{$results} == 1 ) { |
380 |
if (@$results == 1) { |
428 |
my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] ); |
381 |
my $matched_record = C4::Search::new_record_from_zebra( $server, $results->[0] ); |
429 |
SetUTF8Flag($matched_record); |
382 |
SetUTF8Flag($matched_record); |
430 |
$matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid ); |
383 |
$matched_record_id = GetRecordId( $matched_record, $tagid, $subfieldid ); |
Lines 443-451
RECORD: foreach my $record ( @{$marc_records} ) {
Link Here
|
443 |
next; |
396 |
next; |
444 |
} |
397 |
} |
445 |
} |
398 |
} |
446 |
} elsif ( @{$results} > 1 ) { |
399 |
} elsif (@$results > 1) { |
447 |
$logger->debug("More than one match for: $query"); |
400 |
$logger->debug("More than one match for: $query"); |
448 |
next; |
401 |
return; |
449 |
} else { |
402 |
} else { |
450 |
$logger->debug("No match for: $query"); |
403 |
$logger->debug("No match for: $query"); |
451 |
} |
404 |
} |
Lines 455-473
RECORD: foreach my $record ( @{$marc_records} ) {
Link Here
|
455 |
if ( length($keepids) == 3 ) { |
408 |
if ( length($keepids) == 3 ) { |
456 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
409 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
457 |
} else { |
410 |
} else { |
458 |
$storeidfield = |
411 |
$storeidfield = MARC::Field->new( |
459 |
MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
412 |
substr($keepids, 0, 3), "", |
|
|
413 |
"", substr($keepids, 3, 1), $originalid |
414 |
); |
460 |
} |
415 |
} |
461 |
$record->insert_fields_ordered($storeidfield); |
416 |
$record->insert_fields_ordered($storeidfield); |
462 |
$record->delete_field( $record->field($tagid) ); |
417 |
$record->delete_field( $record->field($tagid) ); |
463 |
} |
418 |
} |
464 |
} |
419 |
} |
465 |
|
420 |
|
|
|
421 |
# possibly remove fields if --filter |
466 |
foreach my $stringfilter (@$filters) { |
422 |
foreach my $stringfilter (@$filters) { |
467 |
if ( length($stringfilter) == 3 ) { |
423 |
if ( length($stringfilter) == 3 ) { |
468 |
foreach my $field ( $record->field($stringfilter) ) { |
424 |
foreach my $field ( $record->field($stringfilter) ) { |
469 |
$record->delete_field($field); |
425 |
$record->delete_field($field); |
470 |
$logger->debug( "Removed: ", $field->as_string ); |
426 |
$logger->debug("Removed field $stringfilter") if $verbose; |
471 |
} |
427 |
} |
472 |
} elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) { |
428 |
} elsif ( $stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/ ) { |
473 |
my $removetag = $1; |
429 |
my $removetag = $1; |
Lines 475-756
RECORD: foreach my $record ( @{$marc_records} ) {
Link Here
|
475 |
my $removematch = $3; |
431 |
my $removematch = $3; |
476 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
432 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
477 |
foreach my $field ( $record->field($removetag) ) { |
433 |
foreach my $field ( $record->field($removetag) ) { |
478 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
434 |
$field->delete_subfield( code => $removesubfield, match => $removematch ); |
479 |
$logger->debug( "Potentially removed: ", $field->subfield($removesubfield) ); |
|
|
480 |
} |
435 |
} |
481 |
} |
436 |
} |
482 |
} |
437 |
} |
483 |
} |
438 |
} |
484 |
unless ($test_parameter) { |
439 |
|
485 |
if ($authorities) { |
440 |
# if test only, skip DB changes |
486 |
my $authtypecode = GuessAuthTypeCode( $record, $heading_fields ); |
441 |
if ($test_parameter) { |
487 |
my $authid; |
442 |
return; |
488 |
|
443 |
} |
489 |
if ($matched_record_id) { |
444 |
|
490 |
if ($update) { |
445 |
# handle authorities vs biblios |
491 |
## Authority has an id and is in database: update |
446 |
if ($authorities) { |
492 |
eval { |
447 |
my $authtypecode = GuessAuthTypeCode( $record, $heading_fields ); |
493 |
($authid) = ModAuthority( |
448 |
my $authid; |
494 |
$matched_record_id, $record, $authtypecode, |
449 |
|
495 |
$mod_authority_options, |
450 |
if ($matched_record_id) { |
496 |
); |
451 |
if ($update) { |
497 |
}; |
452 |
## Authority has an id and is in database: update |
498 |
if ($@) { |
453 |
eval { |
499 |
warn "ERROR: Update authority $matched_record_id failed: $@\n"; |
454 |
($authid) = ModAuthority( |
500 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
455 |
$matched_record_id, |
501 |
next RECORD; |
456 |
$record, |
502 |
} else { |
457 |
$authtypecode, |
503 |
printlog( { id => $authid, op => "update", status => "ok" } ) if ($logfile); |
458 |
$mod_authority_options |
504 |
} |
|
|
505 |
} elsif ($logfile) { |
506 |
warn "WARNING: Update authority $originalid skipped"; |
507 |
printlog( |
508 |
{ |
509 |
id => $matched_record_id, |
510 |
op => "update", |
511 |
status => |
512 |
"warning: authority already in database and option -update not enabled, skipping..." |
513 |
} |
514 |
); |
459 |
); |
515 |
} |
460 |
}; |
516 |
} elsif ($insert) { |
|
|
517 |
## An authid is defined but no authority in database: insert |
518 |
eval { ($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options ); }; |
519 |
if ($@) { |
461 |
if ($@) { |
520 |
warn "ERROR: Insert authority $originalid failed: $@\n"; |
462 |
warn "Update authority $matched_record_id failed: $@\n"; |
521 |
printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile); |
463 |
printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if $loghandle; |
522 |
next RECORD; |
464 |
return; |
523 |
} else { |
465 |
} else { |
524 |
printlog( { id => $authid, op => "insert", status => "ok" } ) if ($logfile); |
466 |
printlog({ id => $authid, op => "update", status => "ok" }) if $loghandle; |
|
|
467 |
push @search_engine_record_ids, $authid; |
468 |
push @search_engine_records, $record; |
525 |
} |
469 |
} |
526 |
} else { |
470 |
} else { |
527 |
warn "WARNING: Insert authority $originalid skipped"; |
471 |
warn "Authority $originalid matched but update not enabled.\n"; |
528 |
printlog( |
472 |
printlog({ id => $matched_record_id, op => "update", status => "skipped" }) if $loghandle; |
529 |
{ |
|
|
530 |
id => $originalid, op => "insert", |
531 |
status => "warning : authority not in database and option -insert not enabled, skipping..." |
532 |
} |
533 |
) if ($logfile); |
534 |
} |
473 |
} |
535 |
|
474 |
} elsif ($insert) { |
536 |
if ($yamlfile) { |
475 |
## An authid is defined but no authority in database: insert |
537 |
$yamlhash->{$originalid} = YAMLFileEntry( |
476 |
eval { |
538 |
$record, |
477 |
($authid) = AddAuthority( $record, undef, $authtypecode, $add_authority_options ); |
539 |
$authid, |
478 |
}; |
540 |
1 #@FIXME: Really always updated? |
479 |
if ($@) { |
541 |
); |
480 |
warn "Insert authority $originalid failed: $@\n"; |
|
|
481 |
printlog({ id => $originalid, op => "insert", status => "ERROR" }) if $loghandle; |
482 |
return; |
542 |
} |
483 |
} |
|
|
484 |
printlog({ id => $authid, op => "insert", status => "ok" }) if $loghandle; |
543 |
push @search_engine_record_ids, $authid; |
485 |
push @search_engine_record_ids, $authid; |
544 |
push @search_engine_records, $record; |
486 |
push @search_engine_records, $record; |
545 |
} else { |
487 |
} else { |
546 |
my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id ); |
488 |
warn "Authority $originalid not in DB and --insert not set; skipping.\n"; |
547 |
|
489 |
printlog({ id => $originalid, op => "insert", status => "skipped" }) if $loghandle; |
548 |
# check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter |
490 |
} |
549 |
if ( !$matched_record_id && $isbn_check && $isbn ) { |
|
|
550 |
$sth_isbn->execute($isbn); |
551 |
( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow; |
552 |
} |
553 |
|
491 |
|
554 |
if ( defined $idmapfl && $matched_record_id ) { |
492 |
if ($yamlfile) { |
555 |
if ( $sourcetag < "010" ) { |
493 |
$yamlhash->{$originalid} = YAMLFileEntry( |
556 |
if ( $record->field($sourcetag) ) { |
494 |
$record, |
557 |
my $source = $record->field($sourcetag)->data(); |
495 |
$authid, |
558 |
printf( $idmapfh "%s|%s\n", $source, $matched_record_id ); |
496 |
1 #@FIXME: Really always updated? |
559 |
} |
497 |
); |
560 |
} else { |
498 |
} |
561 |
my $source = $record->subfield( $sourcetag, $sourcesubfield ); |
499 |
} |
562 |
printf( $idmapfh "%s|%s\n", $source, $matched_record_id ); |
500 |
else { |
|
|
501 |
# biblios |
502 |
my ( $biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id ); |
503 |
if ( !$matched_record_id && $isbn_check && $isbn ) { |
504 |
my $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?"); |
505 |
$sth_isbn->execute($isbn); |
506 |
( $matched_record_id, $biblioitemnumber ) = $sth_isbn->fetchrow; |
507 |
} |
508 |
if ( defined $idmapfh && $matched_record_id ) { |
509 |
my $sourceval; |
510 |
if ( $sourcetag < "010" ) { |
511 |
if ( my $f = $record->field($sourcetag) ) { |
512 |
$sourceval = $f->data(); |
563 |
} |
513 |
} |
|
|
514 |
} else { |
515 |
$sourceval = $record->subfield( $sourcetag, $sourcesubfield ); |
516 |
} |
517 |
if ( defined $sourceval ) { |
518 |
printf( $idmapfh "%s|%s\n", $sourceval, $matched_record_id ); |
564 |
} |
519 |
} |
|
|
520 |
} |
565 |
|
521 |
|
566 |
# Create biblio, unless we already have it (either match or ISBN) |
522 |
# Create biblio, unless we already have it (either match or ISBN) |
567 |
if ($matched_record_id) { |
523 |
if ($matched_record_id) { |
568 |
eval { $biblioitemnumber = Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; }; |
524 |
if ($update) { |
569 |
if ($update) { |
525 |
eval { |
570 |
my $success; |
526 |
$biblioitemnumber = |
571 |
eval { |
527 |
Koha::Biblios->find($matched_record_id)->biblioitem->biblioitemnumber; |
572 |
$success = ModBiblio( |
528 |
}; |
573 |
$record, $matched_record_id, GetFrameworkCode($matched_record_id), |
529 |
my $success; |
574 |
$mod_biblio_options |
530 |
eval { |
575 |
); |
531 |
$success = ModBiblio( |
576 |
}; |
532 |
$record, |
577 |
if ($@) { |
533 |
$matched_record_id, |
578 |
warn "ERROR: Update biblio $matched_record_id failed: $@\n"; |
534 |
GetFrameworkCode($matched_record_id), |
579 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
535 |
$mod_biblio_options |
580 |
next RECORD; |
536 |
); |
581 |
} elsif ( !$success ) { |
537 |
}; |
582 |
warn "ERROR: Update biblio $matched_record_id failed for unknown reason"; |
|
|
583 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
584 |
next RECORD; |
585 |
} else { |
586 |
$record_id = $matched_record_id; |
587 |
printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile); |
588 |
} |
589 |
} else { |
590 |
warn "WARNING: Update biblio $originalid skipped"; |
591 |
printlog( |
592 |
{ |
593 |
id => $matched_record_id, op => "update", |
594 |
status => "warning : already in database and option -update not enabled, skipping..." |
595 |
} |
596 |
) if ($logfile); |
597 |
} |
598 |
} elsif ($insert) { |
599 |
my $record_clone = $record->clone(); |
600 |
C4::Biblio::_strip_item_fields($record_clone); |
601 |
eval { ( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ) }; |
602 |
if ($@) { |
538 |
if ($@) { |
603 |
warn "ERROR: Insert biblio $originalid failed: $@\n"; |
539 |
warn "ERROR updating biblio $matched_record_id: $@\n"; |
604 |
printlog( { id => $originalid, op => "insert", status => "ERROR" } ) if ($logfile); |
540 |
printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if $loghandle; |
605 |
next RECORD; |
541 |
return; |
|
|
542 |
} elsif (!$success) { |
543 |
warn "Update biblio $matched_record_id returned no success.\n"; |
544 |
printlog({ id => $matched_record_id, op => "update", status => "ERROR" }) if $loghandle; |
545 |
return; |
606 |
} else { |
546 |
} else { |
607 |
printlog( { id => $originalid, op => "insert", status => "ok" } ) if ($logfile); |
547 |
$record_id = $matched_record_id; |
|
|
548 |
printlog({ id => $record_id, op => "update", status => "ok" }) if $loghandle; |
608 |
} |
549 |
} |
609 |
|
550 |
} else { |
|
|
551 |
warn "Biblio $originalid matched but --update not set; skipping.\n"; |
552 |
printlog({ id => $matched_record_id, op => "update", status => "skipped" }) if $loghandle; |
553 |
return; |
554 |
} |
555 |
} |
556 |
elsif ($insert) { |
557 |
my $record_clone = $record->clone(); |
558 |
C4::Biblio::_strip_item_fields($record_clone); |
559 |
eval { |
560 |
( $record_id, $biblioitemnumber ) = AddBiblio( $record_clone, $framework, $add_biblio_options ); |
561 |
}; |
562 |
if ($@) { |
563 |
warn "Insert biblio $originalid failed: $@\n"; |
564 |
printlog({ id => $originalid, op => "insert", status => "ERROR" }) if $loghandle; |
565 |
return; |
566 |
} else { |
567 |
printlog({ id => $originalid, op => "insert", status => "ok" }) if $loghandle; |
610 |
# If incoming record has bib ids set we need to transfer |
568 |
# If incoming record has bib ids set we need to transfer |
611 |
# new ids from record_clone to incoming record to avoid |
569 |
# new ids from record_clone to incoming record to avoid |
612 |
# working on wrong record (the original record) later on |
570 |
# working on wrong record (the original record) later on |
613 |
# when adding items for example |
571 |
# when adding items for example |
614 |
C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber ); |
572 |
C4::Biblio::_koha_marc_update_bib_ids( $record, $framework, $record_id, $biblioitemnumber ) |
615 |
} else { |
573 |
if $record_id; |
616 |
warn "WARNING: Insert biblio $originalid skipped"; |
|
|
617 |
printlog( |
618 |
{ |
619 |
id => $originalid, op => "insert", |
620 |
status => "warning : biblio not in database and option -insert not enabled, skipping..." |
621 |
} |
622 |
) if ($logfile); |
623 |
next RECORD; |
624 |
} |
574 |
} |
625 |
my $record_has_added_items = 0; |
575 |
} else { |
626 |
if ($record_id) { |
576 |
warn "Biblio $originalid not in DB & --insert not set; skipping.\n"; |
627 |
$yamlhash->{$originalid} = $record_id if $yamlfile; |
577 |
printlog({ id => $originalid, op => "insert", status => "skipped" }) if $loghandle; |
628 |
eval { |
578 |
return; |
629 |
( $itemnumbers_ref, $errors_ref ) = |
579 |
} |
630 |
AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, $framework ); |
|
|
631 |
}; |
632 |
my $error_adding = $@; |
633 |
|
634 |
if ($error_adding) { |
635 |
warn "ERROR: Adding items to bib $record_id failed: $error_adding"; |
636 |
printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile); |
637 |
|
638 |
# if we failed because of an exception, assume that |
639 |
# the MARC columns in biblioitems were not set. |
640 |
next RECORD; |
641 |
} |
642 |
|
643 |
$record_has_added_items = @{$itemnumbers_ref}; |
644 |
|
645 |
if ( $dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } |
646 |
@$errors_ref ) |
647 |
{ |
648 |
# Find the record called 'barcode' |
649 |
my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode'); |
650 |
|
651 |
# Now remove any items that didn't have a duplicate_barcode error, |
652 |
# erase the barcodes on items that did, and re-add those items. |
653 |
my %dupes; |
654 |
foreach my $i ( 0 .. $#{$errors_ref} ) { |
655 |
my $ref = $errors_ref->[$i]; |
656 |
if ( $ref && ( $ref->{error_code} eq 'duplicate_barcode' ) ) { |
657 |
$dupes{ $ref->{item_sequence} } = 1; |
658 |
|
659 |
# Delete the error message because we're going to |
660 |
# retry this one. |
661 |
delete $errors_ref->[$i]; |
662 |
} |
663 |
} |
664 |
my $seq = 0; |
665 |
foreach my $field ( $record->field($tag) ) { |
666 |
$seq++; |
667 |
if ( $dupes{$seq} ) { |
668 |
|
669 |
# Here we remove the barcode |
670 |
$field->delete_subfield( code => $sub ); |
671 |
} else { |
672 |
|
673 |
# otherwise we delete the field because we don't want |
674 |
# two of them |
675 |
$record->delete_fields($field); |
676 |
} |
677 |
} |
678 |
|
580 |
|
679 |
# Now re-add the record as before, adding errors to the prev list |
581 |
if ($record_id) { |
680 |
my $more_errors; |
582 |
my $record_has_added_items = 0; |
681 |
eval { |
583 |
$yamlhash->{$originalid} = $record_id if $yamlfile; |
682 |
( $itemnumbers_ref, $more_errors ) = |
584 |
eval { |
683 |
AddItemBatchFromMarc( $record, $record_id, $biblioitemnumber, '' ); |
585 |
( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( |
684 |
}; |
586 |
$record, |
685 |
if ($@) { |
587 |
$record_id, |
686 |
warn "ERROR: Adding items to bib $record_id failed: $@\n"; |
588 |
$biblioitemnumber, |
687 |
printlog( { id => $record_id, op => "insert items", status => "ERROR" } ) if ($logfile); |
589 |
$framework |
688 |
|
590 |
); |
689 |
# if we failed because of an exception, assume that |
591 |
}; |
690 |
# the MARC columns in biblioitems were not set. |
592 |
my $error_adding = $@; |
691 |
next RECORD; |
593 |
if ($error_adding) { |
|
|
594 |
warn "ERROR adding items to biblio $record_id: $error_adding"; |
595 |
printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if $loghandle; |
596 |
# If we failed because of an exception, assume that |
597 |
# the MARC columns in biblioitems were not set. |
598 |
return; |
599 |
} |
600 |
$record_has_added_items = @{$itemnumbers_ref}; |
601 |
|
602 |
if ( $dedup_barcode |
603 |
&& grep { $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref ) |
604 |
{ |
605 |
# Find the record called 'barcode'. |
606 |
my ( $tag, $sub ) = C4::Biblio::GetMarcFromKohaField('items.barcode'); |
607 |
# Now remove any items that didn't have a duplicate_barcode error, |
608 |
# erase the barcodes on items that did, and re-add those items. |
609 |
my %dupes; |
610 |
foreach my $i ( 0 .. $#{$errors_ref} ) { |
611 |
my $ref = $errors_ref->[$i]; |
612 |
if ( $ref && $ref->{error_code} && $ref->{error_code} eq 'duplicate_barcode' ) { |
613 |
$dupes{ $ref->{item_sequence} } = 1; |
614 |
# Delete the error message because we're going to |
615 |
# retry this one. |
616 |
delete $errors_ref->[$i]; |
692 |
} |
617 |
} |
693 |
$record_has_added_items ||= @{$itemnumbers_ref}; |
618 |
} |
694 |
if ( @{$more_errors} ) { |
619 |
my $seq = 0; |
695 |
push @$errors_ref, @{$more_errors}; |
620 |
foreach my $field ( $record->field($tag) ) { |
|
|
621 |
$seq++; |
622 |
if ( $dupes{$seq} ) { |
623 |
# Here we remove the barcode. |
624 |
$field->delete_subfield( code => $sub ); |
625 |
} else { |
626 |
# Otherwise, we delete the field because we don't want |
627 |
# two of them. |
628 |
$record->delete_fields($field); |
696 |
} |
629 |
} |
697 |
} |
630 |
} |
698 |
|
631 |
|
699 |
if ($record_has_added_items) { |
632 |
# Now re-add the record as before, adding errors to the prev list. |
700 |
printlog( { id => $record_id, op => "insert items", status => "ok" } ) if ($logfile); |
633 |
my $more_errors; |
|
|
634 |
eval { |
635 |
( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( |
636 |
$record, |
637 |
$record_id, |
638 |
$biblioitemnumber, |
639 |
'' |
640 |
); |
641 |
}; |
642 |
if ($@) { |
643 |
warn "ERROR re-adding items biblio $record_id: $@\n"; |
644 |
printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if $loghandle; |
645 |
# If we failed because of an exception, assume that |
646 |
# the MARC columns in biblioitems were not set. |
647 |
return; |
701 |
} |
648 |
} |
702 |
|
649 |
$record_has_added_items ||= @{$itemnumbers_ref}; |
703 |
if ( @{$errors_ref} ) { |
650 |
if (@{$more_errors}) { |
704 |
report_item_errors( $record_id, $errors_ref ); |
651 |
push @$errors_ref, @{$more_errors}; |
705 |
} |
652 |
} |
|
|
653 |
} |
706 |
|
654 |
|
707 |
my $biblio = Koha::Biblios->find($record_id); |
655 |
if ($record_has_added_items) { |
708 |
$record = $biblio->metadata_record( { embed_items => 1 } ); |
656 |
printlog({ id => $record_id, op => "insert items", status => "ok" }) if $loghandle; |
|
|
657 |
} |
658 |
if (@$errors_ref) { |
659 |
report_item_errors( $record_id, $errors_ref ); |
660 |
} |
709 |
|
661 |
|
|
|
662 |
my $biblio_obj = Koha::Biblios->find($record_id); |
663 |
if ($biblio_obj) { |
664 |
my $indexed_record = $biblio_obj->metadata_record({ embed_items => 1 }); |
710 |
push @search_engine_record_ids, $record_id; |
665 |
push @search_engine_record_ids, $record_id; |
711 |
push @search_engine_records, $record; |
666 |
push @search_engine_records, $indexed_record; |
712 |
} |
667 |
} |
713 |
} |
668 |
} |
714 |
if ( $record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total ) { |
669 |
} # end authorities vs biblios |
715 |
$schema->txn_commit; |
670 |
|
716 |
$schema->txn_begin; |
671 |
# If we've hit a commit boundary |
717 |
if ($indexer) { |
672 |
if ( $record_number % $commitnum == 0 ) { |
718 |
$indexer->update_index( \@search_engine_record_ids, \@search_engine_records ) unless $skip_indexing; |
673 |
$schema->txn_commit; |
719 |
if ( C4::Context->preference('AutoLinkBiblios') ) { |
674 |
$schema->txn_begin; |
720 |
foreach my $record (@search_engine_records) { |
675 |
if ($indexer && !$skip_indexing) { |
721 |
BiblioAutoLink( $record, $framework ); |
676 |
$indexer->update_index( \@search_engine_record_ids, \@search_engine_records ); |
722 |
} |
677 |
if (!$authorities && C4::Context->preference('AutoLinkBiblios')) { |
|
|
678 |
foreach my $r (@search_engine_records) { |
679 |
BiblioAutoLink($r, $framework); |
723 |
} |
680 |
} |
724 |
@search_engine_record_ids = (); |
|
|
725 |
@search_engine_records = (); |
726 |
} |
681 |
} |
727 |
} |
682 |
} |
|
|
683 |
@search_engine_record_ids = (); |
684 |
@search_engine_records = (); |
728 |
} |
685 |
} |
729 |
print $record->as_formatted() . "\n" if ( $verbose // 0 ) == 2; |
686 |
} # end sub import_single_record |
730 |
last if $record_number == $number; |
|
|
731 |
} |
732 |
$schema->txn_commit; |
733 |
|
687 |
|
734 |
if ($fk_off) { |
|
|
735 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 1"); |
736 |
} |
737 |
|
688 |
|
738 |
# Restore CataloguingLog and AuthoritiesLog |
689 |
### CHANGED: A function for the older USMARC (non-XML) approach |
739 |
delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; |
690 |
sub parse_nonxml_with_marcbatch { |
740 |
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog}; |
691 |
my ($fh) = @_; |
741 |
|
692 |
|
742 |
my $timeneeded = gettimeofday - $starttime; |
693 |
my $batch = MARC::Batch->new('USMARC', $fh); |
743 |
print "\n$record_number MARC records done in $timeneeded seconds\n"; |
694 |
$batch->warnings_off(); |
744 |
if ($logfile) { |
695 |
$batch->strict_off(); |
745 |
print $loghandle "file : $input_marc_file\n"; |
696 |
|
746 |
print $loghandle "$record_number MARC records done in $timeneeded seconds\n"; |
697 |
# Skip file offset |
747 |
$loghandle->close; |
698 |
if ($offset) { |
|
|
699 |
print "Skipping offset: $offset records\n" if $verbose; |
700 |
$batch->next() while $offset-- > 0; |
701 |
} |
702 |
|
703 |
RECORD: while (1) { |
704 |
my $record; |
705 |
eval { $record = $batch->next() }; |
706 |
if ($@) { |
707 |
warn "Bad MARC record ? : $@\n"; |
708 |
next RECORD; |
709 |
} |
710 |
last unless $record; |
711 |
|
712 |
import_single_record($record); |
713 |
last if ( $number && $record_number == $number ); |
714 |
} |
748 |
} |
715 |
} |
749 |
if ($yamlfile) { |
716 |
|
750 |
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; |
717 |
# This version dynamically removes any marc prefix in each XML element, but uses more memory. |
751 |
print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) ); |
718 |
# sub parse_xml_with_twig { |
|
|
719 |
# my ($file) = @_; |
720 |
|
721 |
# # We'll read offset records if needed, but doing so with Twig means |
722 |
# # we have to skip N <record> elements manually. We'll do a counter: |
723 |
# my $skipcount = $offset; |
724 |
|
725 |
# # We'll parse each <record>, convert to MARC::Record, call import_single_record |
726 |
# my $twig = XML::Twig->new( |
727 |
# twig_handlers => { |
728 |
# 'marc:record' => sub { |
729 |
# print "In handler!\n"; |
730 |
# my ($t, $elt) = @_; |
731 |
|
732 |
# # If offset skipping is not done yet, skip this record |
733 |
# if ($skipcount > 0) { |
734 |
# $skipcount--; |
735 |
# $elt->purge(); |
736 |
# return; |
737 |
# } |
738 |
|
739 |
# # Convert <record> to string |
740 |
# my $xml_chunk = $elt->sprint(); |
741 |
|
742 |
# my $marcrec; |
743 |
# eval { |
744 |
# # If your Koha is MARC21, use 'MARC21' |
745 |
# # or 'UNIMARC' if you have that |
746 |
# $marcrec = MARC::Record->new_from_xml($xml_chunk, 'UTF-8', 'MARC21'); |
747 |
# }; |
748 |
# if ($@ || !$marcrec) { |
749 |
# warn "Skipping bad MARC XML record: $@\n"; |
750 |
# $elt->purge(); |
751 |
# return; |
752 |
# } |
753 |
# print "Looking to call import record sub\n"; |
754 |
# import_single_record($marcrec); |
755 |
|
756 |
# # purge from memory |
757 |
# $elt->purge(); |
758 |
|
759 |
# # if we have a record limit |
760 |
# if ($number && $record_number >= $number) { |
761 |
# $t->finish_now(); |
762 |
# } |
763 |
# } |
764 |
# }, |
765 |
# keep_encoding => 1, |
766 |
# ); |
767 |
|
768 |
# $twig->parsefile($file); |
769 |
# } |
770 |
|
771 |
### CHANGED: A function for streaming MARCXML with XML::Twig, with preprocessing the file. |
772 |
sub parse_xml_with_twig { |
773 |
my ($file) = @_; |
774 |
|
775 |
# We'll read offset records if needed, but doing so with Twig means |
776 |
# we have to skip N <record> elements manually. We'll do a counter: |
777 |
my $skipcount = $offset; |
778 |
|
779 |
# Preprocess the XML file to remove namespaces |
780 |
my $preprocessed_file = 'preprocessed_marc.xml'; |
781 |
open my $in, '<', $file or die "Can't open input file $file: $!"; |
782 |
open my $out, '>', $preprocessed_file or die "Can't open output file $preprocessed_file: $!"; |
783 |
|
784 |
while (<$in>) { |
785 |
s/marc://g; # Remove 'marc:' prefix |
786 |
print $out $_; |
787 |
} |
788 |
|
789 |
close $in; |
790 |
close $out; |
791 |
|
792 |
print "Namespaces removed. Processing preprocessed file: $preprocessed_file\n"; |
793 |
|
794 |
# Now process the preprocessed file |
795 |
my $twig = XML::Twig->new( |
796 |
twig_handlers => { |
797 |
'record' => sub { # No namespace needed anymore |
798 |
my ($t, $elt) = @_; |
799 |
|
800 |
# If offset skipping is not done yet, skip this record |
801 |
if ($skipcount > 0) { |
802 |
$skipcount--; |
803 |
$elt->purge(); |
804 |
return; |
805 |
} |
806 |
|
807 |
# Convert <record> to string |
808 |
my $xml_chunk = $elt->sprint(); |
809 |
|
810 |
my $marcrec; |
811 |
eval { |
812 |
$marcrec = MARC::Record->new_from_xml($xml_chunk, 'UTF-8', 'MARC21'); |
813 |
}; |
814 |
if ($@ || !$marcrec) { |
815 |
warn "Skipping bad MARC XML record: $@\n"; |
816 |
$elt->purge(); |
817 |
return; |
818 |
} |
819 |
|
820 |
import_single_record($marcrec); |
821 |
|
822 |
$elt->purge(); |
823 |
|
824 |
# if we have a record limit |
825 |
if ($number && $record_number >= $number) { |
826 |
$t->finish_now(); |
827 |
} |
828 |
} |
829 |
}, |
830 |
keep_encoding => 1, |
831 |
); |
832 |
|
833 |
$twig->parsefile($preprocessed_file); |
752 |
} |
834 |
} |
753 |
exit 0; |
835 |
|
|
|
836 |
|
837 |
#----------------------------------------------------------------- |
838 |
# Helper subroutines from original script |
839 |
#----------------------------------------------------------------- |
754 |
|
840 |
|
755 |
sub YAMLFileEntry { |
841 |
sub YAMLFileEntry { |
756 |
my ( $record, $record_id, $updated ) = @_; |
842 |
my ( $record, $record_id, $updated ) = @_; |
Lines 859-864
sub get_heading_fields {
Link Here
|
859 |
return $headingfields; |
945 |
return $headingfields; |
860 |
} |
946 |
} |
861 |
|
947 |
|
|
|
948 |
#----------------------------------------------------------------- |
949 |
# MAIN |
950 |
#----------------------------------------------------------------- |
951 |
|
952 |
my $fh = IO::File->new($input_marc_file) |
953 |
or die "Could not open $input_marc_file: $!"; |
954 |
|
955 |
### CHANGED: if the user specified -m=MARCXML or something matching /XML/i, |
956 |
### we do the streaming twig approach. Otherwise, do old MARC::Batch approach. |
957 |
if ( defined $format && $format =~ /XML/i ) { |
958 |
print "XML with Twig call...\n"; |
959 |
parse_xml_with_twig($input_marc_file); |
960 |
} else { |
961 |
parse_nonxml_with_marcbatch($fh); |
962 |
} |
963 |
|
964 |
# Final commit |
965 |
$schema->txn_commit; |
966 |
|
967 |
# Index after importations. |
968 |
if ($indexer && !$skip_indexing) { |
969 |
$indexer->update_index( \@search_engine_record_ids, \@search_engine_records ); |
970 |
if (!$authorities && C4::Context->preference('AutoLinkBiblios')) { |
971 |
foreach my $r (@search_engine_records) { |
972 |
BiblioAutoLink($r, $framework); |
973 |
} |
974 |
} |
975 |
} |
976 |
|
977 |
if ($fk_off) { |
978 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 1"); |
979 |
} |
980 |
|
981 |
# Restore CataloguingLog and AuthoritiesLog |
982 |
delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; |
983 |
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog}; |
984 |
|
985 |
my $timeneeded = gettimeofday - $starttime; |
986 |
print "\n$record_number MARC records done in $timeneeded seconds\n"; |
987 |
if ($logfile) { |
988 |
print $loghandle "file : $input_marc_file\n"; |
989 |
print $loghandle "$record_number MARC records done in $timeneeded seconds\n"; |
990 |
$loghandle->close; |
991 |
} |
992 |
if ($yamlfile) { |
993 |
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; |
994 |
print $yamlfileout Encode::decode_utf8( YAML::XS::Dump($yamlhash) ); |
995 |
} |
996 |
exit 0; |
997 |
|
862 |
=head1 NAME |
998 |
=head1 NAME |
863 |
|
999 |
|
864 |
bulkmarcimport.pl - Import bibliographic/authority records into Koha |
1000 |
bulkmarcimport.pl - Import bibliographic/authority records into Koha |
Lines 1054-1058
this option bad records may kill the job.
Link Here
|
1054 |
|
1190 |
|
1055 |
=back |
1191 |
=back |
1056 |
|
1192 |
|
1057 |
=cut |
1193 |
=cut |
1058 |
|
|
|
1059 |
- |
|
|