|
Lines 23-28
use C4::Biblio qw(
Link Here
|
| 23 |
GetMarcFromKohaField |
23 |
GetMarcFromKohaField |
| 24 |
ModBiblio |
24 |
ModBiblio |
| 25 |
ModBiblioMarc |
25 |
ModBiblioMarc |
|
|
26 |
GetFrameworkCode |
| 27 |
GetMarcBiblio |
| 26 |
); |
28 |
); |
| 27 |
use C4::Koha; |
29 |
use C4::Koha; |
| 28 |
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); |
30 |
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); |
|
Lines 45-66
use Koha::SearchEngine;
Link Here
|
| 45 |
use Koha::SearchEngine::Search; |
47 |
use Koha::SearchEngine::Search; |
| 46 |
|
48 |
|
| 47 |
use open qw( :std :encoding(UTF-8) ); |
49 |
use open qw( :std :encoding(UTF-8) ); |
| 48 |
binmode( STDOUT, ":encoding(UTF-8)" ); |
50 |
binmode(STDOUT, ":encoding(UTF-8)"); |
| 49 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
51 |
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1); |
| 50 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
52 |
my $version; |
| 51 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append ); |
53 |
my $delete; |
| 52 |
my $cleanisbn = 1; |
54 |
my $test_parameter; |
| 53 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
55 |
my $skip_marc8_conversion; |
|
|
56 |
my $char_encoding; |
| 57 |
my $verbose; |
| 58 |
my $commit; |
| 59 |
my $fk_off; |
| 60 |
my $format; |
| 61 |
my $biblios; |
| 62 |
my $authorities; |
| 63 |
my $keepids; |
| 64 |
my $match; |
| 65 |
my $isbn_check; |
| 66 |
my $logfile; |
| 67 |
my $insert; |
| 68 |
my $filters; |
| 69 |
my $update; |
| 70 |
my $all; |
| 71 |
my $yamlfile; |
| 72 |
my $authtypes; |
| 73 |
my $append; |
| 74 |
my $sourcetag; |
| 75 |
my $sourcesubfield; |
| 76 |
my $idmapfl; |
| 77 |
my $dedup_barcode; |
| 54 |
my $framework = ''; |
78 |
my $framework = ''; |
| 55 |
my $localcust; |
79 |
my $localcust; |
| 56 |
my $marc_mod_template = ''; |
80 |
my $marc_mod_template = ''; |
| 57 |
my $marc_mod_template_id = -1; |
81 |
my $marc_mod_template_id = -1; |
| 58 |
|
82 |
$| = 1; |
| 59 |
$|=1; |
|
|
| 60 |
|
83 |
|
| 61 |
GetOptions( |
84 |
GetOptions( |
| 62 |
'commit:f' => \$commit, |
85 |
'commit:f' => \$commit, |
| 63 |
'file:s' => \$input_marc_file, |
86 |
'file:s' => \$input_marc_file, |
| 64 |
'n:f' => \$number, |
87 |
'n:f' => \$number, |
| 65 |
'o|offset:f' => \$offset, |
88 |
'o|offset:f' => \$offset, |
| 66 |
'h' => \$version, |
89 |
'h' => \$version, |
|
Lines 77-98
GetOptions(
Link Here
|
| 77 |
'b|biblios' => \$biblios, |
100 |
'b|biblios' => \$biblios, |
| 78 |
'a|authorities' => \$authorities, |
101 |
'a|authorities' => \$authorities, |
| 79 |
'authtypes:s' => \$authtypes, |
102 |
'authtypes:s' => \$authtypes, |
| 80 |
'filter=s@' => \$filters, |
103 |
'filter=s@' => \$filters, |
| 81 |
'insert' => \$insert, |
104 |
'insert' => \$insert, |
| 82 |
'update' => \$update, |
105 |
'update' => \$update, |
| 83 |
'all' => \$all, |
106 |
'all' => \$all, |
| 84 |
'match=s@' => \$match, |
107 |
'match=s@' => \$match, |
| 85 |
'i|isbn' => \$isbn_check, |
108 |
'i|isbn' => \$isbn_check, |
| 86 |
'x:s' => \$sourcetag, |
109 |
'x:s' => \$sourcetag, |
| 87 |
'y:s' => \$sourcesubfield, |
110 |
'y:s' => \$sourcesubfield, |
| 88 |
'idmap:s' => \$idmapfl, |
111 |
'idmap:s' => \$idmapfl, |
| 89 |
'cleanisbn!' => \$cleanisbn, |
112 |
'cleanisbn!' => \$cleanisbn, |
| 90 |
'yaml:s' => \$yamlfile, |
113 |
'yaml:s' => \$yamlfile, |
| 91 |
'dedupbarcode' => \$dedup_barcode, |
114 |
'dedupbarcode' => \$dedup_barcode, |
| 92 |
'framework=s' => \$framework, |
115 |
'framework=s' => \$framework, |
| 93 |
'custom:s' => \$localcust, |
116 |
'custom:s' => \$localcust, |
| 94 |
'marcmodtemplate:s' => \$marc_mod_template, |
117 |
'marcmodtemplate:s' => \$marc_mod_template, |
| 95 |
); |
118 |
); |
|
|
119 |
|
| 96 |
$biblios ||= !$authorities; |
120 |
$biblios ||= !$authorities; |
| 97 |
$insert ||= !$update; |
121 |
$insert ||= !$update; |
| 98 |
my $writemode = ($append) ? "a" : "w"; |
122 |
my $writemode = ($append) ? "a" : "w"; |
|
Lines 104-109
if ($all) {
Link Here
|
| 104 |
$update = 1; |
128 |
$update = 1; |
| 105 |
} |
129 |
} |
| 106 |
|
130 |
|
|
|
131 |
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch'); |
| 132 |
my $modify_biblio_marc_options = { |
| 133 |
defer_search_engine_indexing => $using_elastic_search, |
| 134 |
overlay_context => { source => 'bulkmarcimport' } |
| 135 |
}; |
| 136 |
|
| 137 |
my @search_engine_record_ids; |
| 138 |
my @search_engine_records; |
| 139 |
my $indexer; |
| 140 |
if ($using_elastic_search) { |
| 141 |
use Koha::SearchEngine::Elasticsearch::Indexer; |
| 142 |
$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( |
| 143 |
{ index => $authorities ? |
| 144 |
$Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX : |
| 145 |
$Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX |
| 146 |
} |
| 147 |
); |
| 148 |
} |
| 149 |
|
| 107 |
if ($version || ($input_marc_file eq '')) { |
150 |
if ($version || ($input_marc_file eq '')) { |
| 108 |
pod2usage( -verbose => 2 ); |
151 |
pod2usage( -verbose => 2 ); |
| 109 |
exit; |
152 |
exit; |
|
Lines 112-124
if( $update && !( $match || $isbn_check ) ) {
Link Here
|
| 112 |
warn "Using -update without -match or -isbn seems to be useless.\n"; |
155 |
warn "Using -update without -match or -isbn seems to be useless.\n"; |
| 113 |
} |
156 |
} |
| 114 |
|
157 |
|
| 115 |
if(defined $localcust) { #local customize module |
158 |
if (defined $localcust) { #local customize module |
| 116 |
if(!-e $localcust) { |
159 |
if (!-e $localcust) { |
| 117 |
$localcust= $localcust||'LocalChanges'; #default name |
160 |
$localcust = $localcust || 'LocalChanges'; #default name |
| 118 |
$localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only |
161 |
$localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only |
| 119 |
$localcust=~ s/\.pm$//; #remove extension |
162 |
$localcust =~ s/\.pm$//; #remove extension |
| 120 |
my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir |
163 |
my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir |
| 121 |
if(-e $fqcust) { |
164 |
if (-e $fqcust) { |
| 122 |
$localcust= $fqcust; |
165 |
$localcust= $fqcust; |
| 123 |
} |
166 |
} |
| 124 |
else { |
167 |
else { |
|
Lines 127-133
if(defined $localcust) { #local customize module
Link Here
|
| 127 |
} |
170 |
} |
| 128 |
} |
171 |
} |
| 129 |
require $localcust if $localcust; |
172 |
require $localcust if $localcust; |
| 130 |
$localcust=\&customize if $localcust; |
173 |
$localcust = \&customize if $localcust; |
| 131 |
} |
174 |
} |
| 132 |
|
175 |
|
| 133 |
if($marc_mod_template ne '') { |
176 |
if($marc_mod_template ne '') { |
|
Lines 152-219
if($marc_mod_template ne '') {
Link Here
|
| 152 |
} |
195 |
} |
| 153 |
|
196 |
|
| 154 |
my $dbh = C4::Context->dbh; |
197 |
my $dbh = C4::Context->dbh; |
| 155 |
my $heading_fields=get_heading_fields(); |
198 |
my $heading_fields = get_heading_fields(); |
| 156 |
|
|
|
| 157 |
my $idmapfh; |
199 |
my $idmapfh; |
|
|
200 |
|
| 158 |
if (defined $idmapfl) { |
201 |
if (defined $idmapfl) { |
| 159 |
open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n"; |
202 |
open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n"; |
| 160 |
} |
203 |
} |
| 161 |
|
204 |
|
| 162 |
if ((not defined $sourcesubfield) && (not defined $sourcetag)){ |
205 |
if ((not defined $sourcesubfield) && (not defined $sourcetag)) { |
| 163 |
$sourcetag="910"; |
206 |
$sourcetag = "910"; |
| 164 |
$sourcesubfield="a"; |
207 |
$sourcesubfield = "a"; |
| 165 |
} |
208 |
} |
| 166 |
|
209 |
|
| 167 |
|
|
|
| 168 |
# Disable logging for the biblios and authorities import operation. It would unnecessarily |
210 |
# Disable logging for the biblios and authorities import operation. It would unnecessarily |
| 169 |
# slow the import |
211 |
# slow the import |
| 170 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
212 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
| 171 |
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0; |
213 |
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0; |
| 172 |
|
214 |
|
| 173 |
if ($fk_off) { |
215 |
if ($fk_off) { |
| 174 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 0"); |
216 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 0"); |
| 175 |
} |
217 |
} |
| 176 |
|
218 |
|
| 177 |
|
|
|
| 178 |
if ($delete) { |
219 |
if ($delete) { |
| 179 |
if ($biblios){ |
220 |
if ($biblios) { |
| 180 |
print "deleting biblios\n"; |
221 |
print "Deleting biblios\n"; |
| 181 |
$dbh->do("DELETE FROM biblio"); |
222 |
$dbh->do("DELETE FROM biblio"); |
| 182 |
$dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1"); |
223 |
$dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1"); |
| 183 |
$dbh->do("DELETE FROM biblioitems"); |
224 |
$dbh->do("DELETE FROM biblioitems"); |
| 184 |
$dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1"); |
225 |
$dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1"); |
| 185 |
$dbh->do("DELETE FROM items"); |
226 |
$dbh->do("DELETE FROM items"); |
| 186 |
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); |
227 |
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); |
| 187 |
} |
228 |
} |
| 188 |
else { |
229 |
else { |
| 189 |
print "deleting authorities\n"; |
230 |
print "Deleting authorities\n"; |
| 190 |
$dbh->do("truncate auth_header"); |
231 |
$dbh->do("truncate auth_header"); |
| 191 |
} |
232 |
} |
| 192 |
$dbh->do("truncate zebraqueue"); |
233 |
$dbh->do("truncate zebraqueue"); |
| 193 |
} |
234 |
} |
| 194 |
|
235 |
|
| 195 |
|
|
|
| 196 |
|
| 197 |
if ($test_parameter) { |
236 |
if ($test_parameter) { |
| 198 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
237 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
| 199 |
} |
238 |
} |
| 200 |
|
239 |
|
| 201 |
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21'; |
240 |
my $batch; |
|
|
241 |
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21'; |
| 202 |
|
242 |
|
| 203 |
# The definition of $searcher must be before MARC::Batch->new |
243 |
# The definition of $searcher must be before MARC::Batch->new |
| 204 |
my $searcher = Koha::SearchEngine::Search->new( |
244 |
my $searcher = Koha::SearchEngine::Search->new( |
| 205 |
{ |
245 |
{ |
| 206 |
index => ( |
246 |
index => ( |
| 207 |
$authorities |
247 |
$authorities |
| 208 |
? $Koha::SearchEngine::AUTHORITIES_INDEX |
248 |
? $Koha::SearchEngine::AUTHORITIES_INDEX |
| 209 |
: $Koha::SearchEngine::BIBLIOS_INDEX |
249 |
: $Koha::SearchEngine::BIBLIOS_INDEX |
| 210 |
) |
250 |
) |
| 211 |
} |
251 |
} |
| 212 |
); |
252 |
); |
| 213 |
|
253 |
|
| 214 |
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose; |
254 |
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose; |
| 215 |
my $starttime = gettimeofday; |
255 |
my $starttime = gettimeofday; |
| 216 |
my $batch; |
256 |
|
| 217 |
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
257 |
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
| 218 |
if (defined $format && $format =~ /XML/i) { |
258 |
if (defined $format && $format =~ /XML/i) { |
| 219 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
259 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
|
Lines 226-279
if (defined $format && $format =~ /XML/i) {
Link Here
|
| 226 |
# extract the records, not using regexes to look |
266 |
# extract the records, not using regexes to look |
| 227 |
# for <record>.*</record>. |
267 |
# for <record>.*</record>. |
| 228 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
268 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
| 229 |
my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour)); |
269 |
my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour)); |
| 230 |
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. |
270 |
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. |
| 231 |
$recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21"); |
271 |
$recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21"); |
| 232 |
$MARC::File::XML::_load_args{RecordFormat} = $recordformat; |
272 |
$MARC::File::XML::_load_args{RecordFormat} = $recordformat; |
| 233 |
$batch = MARC::Batch->new( 'XML', $fh ); |
273 |
$batch = MARC::Batch->new('XML', $fh); |
| 234 |
} else { |
274 |
} |
| 235 |
$batch = MARC::Batch->new( 'USMARC', $fh ); |
275 |
else { |
|
|
276 |
$batch = MARC::Batch->new('USMARC', $fh); |
| 236 |
} |
277 |
} |
|
|
278 |
|
| 237 |
$batch->warnings_off(); |
279 |
$batch->warnings_off(); |
| 238 |
$batch->strict_off(); |
280 |
$batch->strict_off(); |
| 239 |
my $i=0; |
|
|
| 240 |
my $commitnum = $commit ? $commit : 50; |
281 |
my $commitnum = $commit ? $commit : 50; |
| 241 |
my $yamlhash; |
282 |
my $yamlhash; |
| 242 |
|
283 |
|
| 243 |
# Skip file offset |
284 |
# Skip file offset |
| 244 |
if ( $offset ) { |
285 |
if ($offset) { |
| 245 |
print "Skipping file offset: $offset records\n"; |
286 |
print "Skipping file offset: $offset records\n"; |
| 246 |
$batch->next() while ($offset--); |
287 |
$batch->next() while ($offset--); |
| 247 |
} |
288 |
} |
| 248 |
|
289 |
|
| 249 |
my ($tagid,$subfieldid); |
290 |
my ($tagid, $subfieldid); |
| 250 |
if ($authorities){ |
291 |
if ($authorities) { |
| 251 |
$tagid='001'; |
292 |
$tagid = '001'; |
| 252 |
} |
293 |
} |
| 253 |
else { |
294 |
else { |
| 254 |
( $tagid, $subfieldid ) = |
295 |
($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber"); |
| 255 |
GetMarcFromKohaField( "biblio.biblionumber" ); |
296 |
$tagid ||= "001"; |
| 256 |
$tagid||="001"; |
|
|
| 257 |
} |
297 |
} |
| 258 |
|
298 |
|
|
|
299 |
my $sth_isbn; |
| 259 |
# the SQL query to search on isbn |
300 |
# the SQL query to search on isbn |
| 260 |
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?"); |
301 |
if ($isbn_check) { |
|
|
302 |
$sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?"); |
| 303 |
} |
| 261 |
|
304 |
|
| 262 |
my $loghandle; |
305 |
my $loghandle; |
| 263 |
if ($logfile){ |
306 |
if ($logfile) { |
| 264 |
$loghandle= IO::File->new($logfile, $writemode) ; |
307 |
$loghandle= IO::File->new($logfile, $writemode); |
| 265 |
print $loghandle "id;operation;status\n"; |
308 |
print $loghandle "id;operation;status\n"; |
| 266 |
} |
309 |
} |
| 267 |
|
310 |
|
|
|
311 |
my $record_number = 0; |
| 268 |
my $logger = Koha::Logger->get; |
312 |
my $logger = Koha::Logger->get; |
| 269 |
my $schema = Koha::Database->schema; |
313 |
my $schema = Koha::Database->schema; |
| 270 |
$schema->txn_begin; |
314 |
my $marc_records = []; |
| 271 |
RECORD: while ( ) { |
315 |
RECORD: while ( ) { |
| 272 |
my $record; |
316 |
my $record; |
| 273 |
# get records |
317 |
$record_number++; |
|
|
318 |
# get record |
| 274 |
eval { $record = $batch->next() }; |
319 |
eval { $record = $batch->next() }; |
| 275 |
if ( $@ ) { |
320 |
if ($@) { |
| 276 |
print "Bad MARC record $i: $@ skipped\n"; |
321 |
print "Bad MARC record $record_number: $@ skipped\n"; |
| 277 |
# FIXME - because MARC::Batch->next() combines grabbing the next |
322 |
# FIXME - because MARC::Batch->next() combines grabbing the next |
| 278 |
# blob and parsing it into one operation, a correctable condition |
323 |
# blob and parsing it into one operation, a correctable condition |
| 279 |
# such as a MARC-8 record claiming that it's UTF-8 can't be recovered |
324 |
# such as a MARC-8 record claiming that it's UTF-8 can't be recovered |
|
Lines 282-560
RECORD: while ( ) {
Link Here
|
| 282 |
# C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. |
327 |
# C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. |
| 283 |
next; |
328 |
next; |
| 284 |
} |
329 |
} |
| 285 |
# skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure |
330 |
if ($record) { |
| 286 |
last unless ( $record ); |
331 |
# transcode the record to UTF8 if needed & applicable. |
| 287 |
$i++; |
332 |
if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { |
| 288 |
if( ($verbose//1)==1 ) { #no dot for verbose==2 |
333 |
my ($guessed_charset, $charset_errors); |
| 289 |
print "." . ( $i % 100==0 ? "\n$i" : '' ); |
334 |
($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : '')); |
|
|
335 |
if ($guessed_charset eq 'failed') { |
| 336 |
warn "ERROR: failed to perform character conversion for record $record_number\n"; |
| 337 |
next RECORD; |
| 338 |
} |
| 339 |
} |
| 340 |
SetUTF8Flag($record); |
| 341 |
&$localcust($record) if $localcust; |
| 342 |
push @{$marc_records}, $record; |
| 343 |
} |
| 344 |
else { |
| 345 |
last; |
| 290 |
} |
346 |
} |
|
|
347 |
} |
| 291 |
|
348 |
|
| 292 |
# transcode the record to UTF8 if needed & applicable. |
349 |
$record_number = 0; |
| 293 |
if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { |
350 |
my $records_total = @{$marc_records}; |
| 294 |
# FIXME update condition |
351 |
$schema->txn_begin; |
| 295 |
my ($guessed_charset, $charset_errors); |
352 |
RECORD: foreach my $record (@{$marc_records}) { |
| 296 |
($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':'')); |
353 |
$record_number++; |
| 297 |
if ($guessed_charset eq 'failed') { |
354 |
if (($verbose//1) == 1) { #no dot for verbose==2 |
| 298 |
warn "ERROR: failed to perform character conversion for record $i\n"; |
355 |
print "." . ($record_number % 100 == 0 ? "\n$record_number" : ''); |
| 299 |
next RECORD; |
|
|
| 300 |
} |
| 301 |
} |
356 |
} |
| 302 |
SetUTF8Flag($record); |
357 |
|
| 303 |
if($marc_mod_template_id > 0) { |
358 |
if ($marc_mod_template_id > 0) { |
| 304 |
print "Modifying MARC\n" if $verbose; |
359 |
print "Modifying MARC\n" if $verbose; |
| 305 |
ModifyRecordWithTemplate( $marc_mod_template_id, $record ); |
360 |
ModifyRecordWithTemplate( $marc_mod_template_id, $record ); |
| 306 |
} |
361 |
} |
| 307 |
&$localcust($record) if $localcust; |
362 |
|
| 308 |
my $isbn; |
363 |
my $isbn; |
| 309 |
# remove trailing - in isbn (only for biblios, of course) |
364 |
# remove trailing - in isbn (only for biblios, of course) |
| 310 |
if( $biblios ) { |
365 |
if ($biblios && ($cleanisbn || $isbn_check)) { |
| 311 |
my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020'; |
366 |
my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; |
| 312 |
my $field = $record->field($tag); |
367 |
my $field = $record->field($tag); |
| 313 |
$isbn = $field && $field->subfield('a'); |
368 |
$isbn = $field && $field->subfield('a'); |
| 314 |
if ( $isbn && $cleanisbn ) { |
369 |
if ($isbn && $cleanisbn) { |
| 315 |
$isbn =~ s/-//g; |
370 |
$isbn =~ s/-//g; |
| 316 |
$field->update('a' => $isbn); |
371 |
$field->update('a' => $isbn); |
| 317 |
} |
372 |
} |
| 318 |
} |
373 |
} |
| 319 |
my $id; |
|
|
| 320 |
# search for duplicates (based on Local-number) |
374 |
# search for duplicates (based on Local-number) |
| 321 |
my $originalid; |
375 |
my $originalid = GetRecordId($record, $tagid, $subfieldid); |
| 322 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
376 |
my $matched_record_id = undef; |
| 323 |
if ($match) { |
377 |
if ($match) { |
| 324 |
require C4::Search; |
378 |
require C4::Search; |
| 325 |
my $query = build_query( $match, $record ); |
379 |
my $server = ($authorities ? 'authorityserver' : 'biblioserver'); |
| 326 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
380 |
my $query = build_query($match, $record); |
| 327 |
my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] ); |
381 |
$logger->debug("Bulkmarcimport: $query"); |
|
|
382 |
my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]); |
| 328 |
# changed to warn so able to continue with one broken record |
383 |
# changed to warn so able to continue with one broken record |
| 329 |
if ( defined $error ) { |
384 |
if (defined $error) { |
| 330 |
warn "unable to search the database for duplicates : $error"; |
385 |
warn "unable to search the database for duplicates : $error"; |
| 331 |
printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile); |
386 |
printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile); |
| 332 |
next RECORD; |
387 |
next RECORD; |
| 333 |
} |
388 |
} |
| 334 |
if ( $results && scalar(@$results) == 1 ) { |
389 |
$logger->debug("Bulkmarcimport: $query $server : $totalhits"); |
| 335 |
my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] ); |
390 |
# sub SimpleSearch could return undefined, but only on error, so |
| 336 |
SetUTF8Flag($marcrecord); |
391 |
# should not really need to safeguard here, but do so anyway |
| 337 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
392 |
$results //= []; |
| 338 |
if ( $authorities && $marcFlavour ) { |
393 |
if (@{$results} == 1) { |
| 339 |
#Skip if authority in database is the same as the on in database |
394 |
my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]); |
| 340 |
if ( $marcrecord->field('005') && $record->field('005') && |
395 |
SetUTF8Flag($matched_record); |
| 341 |
$marcrecord->field('005')->data && $record->field('005')->data && |
396 |
$matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid); |
| 342 |
$marcrecord->field('005')->data >= $record->field('005')->data ) { |
397 |
|
|
|
398 |
if ($authorities && $marc_flavour) { |
| 399 |
#Skip if authority in database is the same or newer than the incoming record |
| 400 |
if (RecordRevisionIsGtOrEq($matched_record, $record)) { |
| 343 |
if ($yamlfile) { |
401 |
if ($yamlfile) { |
| 344 |
$yamlhash->{$originalid}->{'authid'} = $id; |
402 |
$yamlhash->{$originalid} = YAMLFileEntry( |
| 345 |
|
403 |
$matched_record, |
| 346 |
# we recover all subfields of the heading authorities |
404 |
$matched_record_id, |
| 347 |
my @subfields; |
405 |
0 |
| 348 |
foreach my $field ( $marcrecord->field("2..") ) { |
406 |
); |
| 349 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
|
|
| 350 |
} |
| 351 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
| 352 |
$yamlhash->{$originalid}->{'updated'} = 0; |
| 353 |
} |
407 |
} |
| 354 |
next; |
408 |
next; |
| 355 |
} |
409 |
} |
| 356 |
} |
410 |
} |
| 357 |
} elsif ( $results && scalar(@$results) > 1 ) { |
|
|
| 358 |
$logger->debug("more than one match for $query"); |
| 359 |
} else { |
| 360 |
$logger->debug("nomatch for $query"); |
| 361 |
} |
411 |
} |
| 362 |
} |
412 |
elsif(@{$results} > 1) { |
| 363 |
if ($keepids && $originalid) { |
413 |
$logger->debug("More than one match for: $query"); |
|
|
414 |
} |
| 415 |
else { |
| 416 |
$logger->debug("No match for: $query"); |
| 417 |
} |
| 418 |
|
| 419 |
if ($keepids && $originalid) { |
| 364 |
my $storeidfield; |
420 |
my $storeidfield; |
| 365 |
if ( length($keepids) == 3 ) { |
421 |
if (length($keepids) == 3) { |
| 366 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
422 |
$storeidfield = MARC::Field->new($keepids, $originalid); |
| 367 |
} else { |
423 |
} else { |
| 368 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
424 |
$storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid); |
| 369 |
} |
425 |
} |
| 370 |
$record->insert_fields_ordered($storeidfield); |
426 |
$record->insert_fields_ordered($storeidfield); |
| 371 |
$record->delete_field( $record->field($tagid) ); |
427 |
$record->delete_field($record->field($tagid)); |
|
|
428 |
} |
| 372 |
} |
429 |
} |
|
|
430 |
|
| 373 |
foreach my $stringfilter (@$filters) { |
431 |
foreach my $stringfilter (@$filters) { |
| 374 |
if ( length($stringfilter) == 3 ) { |
432 |
if (length($stringfilter) == 3) { |
| 375 |
foreach my $field ( $record->field($stringfilter) ) { |
433 |
foreach my $field ($record->field($stringfilter)) { |
| 376 |
$record->delete_field($field); |
434 |
$record->delete_field($field); |
| 377 |
$logger->debug("removed : ", $field->as_string); |
435 |
$logger->debug("Removed: ", $field->as_string); |
| 378 |
} |
436 |
} |
| 379 |
} elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { |
437 |
} elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { |
| 380 |
my $removetag = $1; |
438 |
my $removetag = $1; |
| 381 |
my $removesubfield = $2; |
439 |
my $removesubfield = $2; |
| 382 |
my $removematch = $3; |
440 |
my $removematch = $3; |
| 383 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
441 |
if (($removetag > "010") && $removesubfield) { |
| 384 |
foreach my $field ( $record->field($removetag) ) { |
442 |
foreach my $field ($record->field($removetag)) { |
| 385 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
443 |
$field->delete_subfield(code => "$removesubfield", match => $removematch); |
| 386 |
$logger->debug("Potentially removed : ", $field->subfield($removesubfield)); |
444 |
$logger->debug("Potentially removed: ", $field->subfield($removesubfield)); |
| 387 |
} |
445 |
} |
| 388 |
} |
446 |
} |
| 389 |
} |
447 |
} |
| 390 |
} |
448 |
} |
| 391 |
unless ($test_parameter) { |
449 |
unless ($test_parameter) { |
| 392 |
if ($authorities){ |
450 |
if ($authorities) { |
| 393 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
451 |
my $authtypecode = GuessAuthTypeCode($record, $heading_fields); |
| 394 |
my $authid= ($id?$id:GuessAuthId($record)); |
452 |
my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record)); |
| 395 |
if ($authid && GetAuthority($authid) && $update ){ |
453 |
if ($authid && GetAuthority($authid) && $update) { |
| 396 |
## Authority has an id and is in database : Replace |
454 |
## Authority has an id and is in database : Replace |
| 397 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
455 |
eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) }; |
| 398 |
if ($@){ |
456 |
if ($@) { |
| 399 |
warn "Problem with authority $authid Cannot Modify"; |
457 |
warn "Problem with authority $authid Cannot Modify"; |
| 400 |
printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile); |
458 |
printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile); |
| 401 |
} |
459 |
} |
| 402 |
else{ |
460 |
else{ |
| 403 |
printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile); |
461 |
printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile); |
| 404 |
} |
462 |
} |
| 405 |
} |
463 |
} |
| 406 |
else { |
464 |
else { |
| 407 |
## True insert in database |
465 |
## True insert in database |
| 408 |
eval { ( $authid ) = AddAuthority($record,"", $authtypecode) }; |
466 |
eval { ( $authid ) = AddAuthority($record, "", $authtypecode) }; |
| 409 |
if ($@){ |
467 |
if ($@) { |
| 410 |
warn "Problem with authority $authid Cannot Add".$@; |
468 |
warn "Problem with authority $originalid Cannot Add ".$@; |
| 411 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile); |
469 |
printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile); |
| 412 |
} |
470 |
} |
| 413 |
else{ |
471 |
else { |
| 414 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
472 |
printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile); |
| 415 |
} |
473 |
} |
| 416 |
} |
474 |
|
| 417 |
if ($yamlfile) { |
|
|
| 418 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
| 419 |
my @subfields; |
| 420 |
foreach my $field ( $record->field("2..") ) { |
| 421 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
| 422 |
} |
475 |
} |
| 423 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
476 |
if ($yamlfile) { |
| 424 |
$yamlhash->{$originalid}->{'updated'} = 1; |
477 |
$yamlhash->{$originalid} = YAMLFileEntry( |
|
|
478 |
$record, |
| 479 |
$authid, |
| 480 |
1 #@FIXME: Really always updated? |
| 481 |
); |
| 425 |
} |
482 |
} |
| 426 |
} |
483 |
} |
| 427 |
else { |
484 |
else { |
| 428 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
485 |
my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id); |
| 429 |
$biblionumber = $id; |
|
|
| 430 |
# check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter |
486 |
# check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter |
| 431 |
if (!$biblionumber && $isbn_check && $isbn) { |
487 |
if (!$matched_record_id && $isbn_check && $isbn) { |
| 432 |
# warn "search ISBN : $isbn"; |
|
|
| 433 |
$sth_isbn->execute($isbn); |
488 |
$sth_isbn->execute($isbn); |
| 434 |
($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow; |
489 |
($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow; |
|
|
490 |
} |
| 491 |
|
| 492 |
if (defined $idmapfl && $matched_record_id) { |
| 493 |
if ($sourcetag < "010") { |
| 494 |
if ($record->field($sourcetag)) { |
| 495 |
my $source = $record->field($sourcetag)->data(); |
| 496 |
printf($idmapfh "%s|%s\n", $source, $matched_record_id); |
| 497 |
} |
| 498 |
} |
| 499 |
else { |
| 500 |
my $source = $record->subfield($sourcetag, $sourcesubfield); |
| 501 |
printf($idmapfh "%s|%s\n", $source, $matched_record_id); |
| 502 |
} |
| 435 |
} |
503 |
} |
| 436 |
if (defined $idmapfl) { |
504 |
|
| 437 |
if ($sourcetag < "010"){ |
505 |
# create biblio, unless we already have it ( either match or isbn ) |
| 438 |
if ($record->field($sourcetag)){ |
506 |
if ($matched_record_id) { |
| 439 |
my $source = $record->field($sourcetag)->data(); |
507 |
# TODO: Implement also for authority records! |
| 440 |
printf($idmapfh "%s|%s\n",$source,$biblionumber); |
|
|
| 441 |
} |
| 442 |
} else { |
| 443 |
my $source=$record->subfield($sourcetag,$sourcesubfield); |
| 444 |
printf($idmapfh "%s|%s\n",$source,$biblionumber); |
| 445 |
} |
| 446 |
} |
| 447 |
# create biblio, unless we already have it ( either match or isbn ) |
| 448 |
if ($biblionumber) { |
| 449 |
eval{ |
508 |
eval{ |
| 450 |
$biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber; |
509 |
$biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber; |
| 451 |
}; |
510 |
}; |
| 452 |
if ($update) { |
511 |
if ($update) { |
| 453 |
eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) }; |
512 |
my $success; |
|
|
513 |
eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) }; |
| 454 |
if ($@) { |
514 |
if ($@) { |
| 455 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
515 |
warn "ERROR: Edit biblio $matched_record_id failed: $@\n"; |
| 456 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
516 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
| 457 |
next RECORD; |
517 |
next RECORD; |
| 458 |
} else { |
|
|
| 459 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
| 460 |
} |
518 |
} |
| 461 |
} else { |
519 |
elsif (!$success) { |
| 462 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
520 |
warn "ERROR: Edit biblio $matched_record_id failed for unkown reason"; |
| 463 |
} |
521 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
| 464 |
} else { |
|
|
| 465 |
if ($insert) { |
| 466 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) }; |
| 467 |
if ($@) { |
| 468 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
| 469 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
| 470 |
next RECORD; |
522 |
next RECORD; |
| 471 |
} else { |
|
|
| 472 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
| 473 |
} |
523 |
} |
| 474 |
} else { |
524 |
else { |
| 475 |
warn "WARNING: Updating record ".($id||$originalid)." failed"; |
525 |
$record_id = $matched_record_id; |
| 476 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
526 |
printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile); |
|
|
527 |
} |
| 528 |
} |
| 529 |
else { |
| 530 |
printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile); |
| 531 |
} |
| 532 |
} |
| 533 |
elsif ($insert) { |
| 534 |
eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; |
| 535 |
if ($@) { |
| 536 |
warn "ERROR: Adding biblio $record_id failed: $@\n"; |
| 537 |
printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile); |
| 477 |
next RECORD; |
538 |
next RECORD; |
| 478 |
} |
539 |
} |
|
|
540 |
else { |
| 541 |
printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile); |
| 542 |
} |
| 479 |
} |
543 |
} |
| 480 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
544 |
else { |
| 481 |
my $error_adding = $@; |
545 |
warn "WARNING: Updating record ".($originalid)." failed"; |
| 482 |
# Work on a clone so that if there are real errors, we can maybe |
546 |
printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile); |
| 483 |
# fix them up later. |
|
|
| 484 |
my $clone_record = $record->clone(); |
| 485 |
C4::Biblio::_strip_item_fields($clone_record, ''); |
| 486 |
# This sets the marc fields if there was an error, and also calls |
| 487 |
# defer_marc_save. |
| 488 |
ModBiblioMarc( $clone_record, $biblionumber ); |
| 489 |
if ( $error_adding ) { |
| 490 |
warn "ERROR: Adding items to bib $biblionumber failed: $error_adding"; |
| 491 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile); |
| 492 |
# if we failed because of an exception, assume that |
| 493 |
# the MARC columns in biblioitems were not set. |
| 494 |
next RECORD; |
547 |
next RECORD; |
| 495 |
} |
548 |
} |
| 496 |
else{ |
549 |
my $record_has_added_items = 0; |
| 497 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile); |
550 |
if ($record_id) { |
| 498 |
} |
551 |
$yamlhash->{$originalid} = $record_id if $yamlfile; |
| 499 |
if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) { |
552 |
eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); }; |
| 500 |
# Find the record called 'barcode' |
553 |
$record_has_added_items = @{$itemnumbers_ref}; |
| 501 |
my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' ); |
554 |
my $error_adding = $@; |
| 502 |
# Now remove any items that didn't have a duplicate_barcode error, |
555 |
# Work on a clone so that if there are real errors, we can maybe |
| 503 |
# erase the barcodes on items that did, and re-add those items. |
556 |
# fix them up later. |
| 504 |
my %dupes; |
557 |
my $clone_record = $record->clone(); |
| 505 |
foreach my $i (0 .. $#{$errors_ref}) { |
558 |
C4::Biblio::_strip_item_fields($clone_record, $framework); |
| 506 |
my $ref = $errors_ref->[$i]; |
559 |
# This sets the marc fields if there was an error, and also calls |
| 507 |
if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) { |
560 |
# defer_marc_save. |
| 508 |
$dupes{$ref->{item_sequence}} = 1; |
561 |
ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options); |
| 509 |
# Delete the error message because we're going to |
562 |
if ($error_adding) { |
| 510 |
# retry this one. |
563 |
warn "ERROR: Adding items to bib $record_id failed: $error_adding"; |
| 511 |
delete $errors_ref->[$i]; |
564 |
printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile); |
| 512 |
} |
|
|
| 513 |
} |
| 514 |
my $seq = 0; |
| 515 |
foreach my $field ($record->field($tag)) { |
| 516 |
$seq++; |
| 517 |
if ($dupes{$seq}) { |
| 518 |
# Here we remove the barcode |
| 519 |
$field->delete_subfield(code => $sub); |
| 520 |
} else { |
| 521 |
# otherwise we delete the field because we don't want |
| 522 |
# two of them |
| 523 |
$record->delete_fields($field); |
| 524 |
} |
| 525 |
} |
| 526 |
# Now re-add the record as before, adding errors to the prev list |
| 527 |
my $more_errors; |
| 528 |
eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
| 529 |
if ( $@ ) { |
| 530 |
warn "ERROR: Adding items to bib $biblionumber failed: $@\n"; |
| 531 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile); |
| 532 |
# if we failed because of an exception, assume that |
565 |
# if we failed because of an exception, assume that |
| 533 |
# the MARC columns in biblioitems were not set. |
566 |
# the MARC columns in biblioitems were not set. |
| 534 |
ModBiblioMarc( $record, $biblionumber ); |
|
|
| 535 |
next RECORD; |
567 |
next RECORD; |
| 536 |
} else { |
|
|
| 537 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile); |
| 538 |
} |
568 |
} |
| 539 |
push @$errors_ref, @{ $more_errors }; |
569 |
else { |
| 540 |
} |
570 |
printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile); |
| 541 |
if ($#{ $errors_ref } > -1) { |
571 |
} |
| 542 |
report_item_errors($biblionumber, $errors_ref); |
572 |
if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) { |
|
|
573 |
# Find the record called 'barcode' |
| 574 |
my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode'); |
| 575 |
# Now remove any items that didn't have a duplicate_barcode error, |
| 576 |
# erase the barcodes on items that did, and re-add those items. |
| 577 |
my %dupes; |
| 578 |
foreach my $i (0 .. $#{$errors_ref}) { |
| 579 |
my $ref = $errors_ref->[$i]; |
| 580 |
if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) { |
| 581 |
$dupes{$ref->{item_sequence}} = 1; |
| 582 |
# Delete the error message because we're going to |
| 583 |
# retry this one. |
| 584 |
delete $errors_ref->[$i]; |
| 585 |
} |
| 586 |
} |
| 587 |
my $seq = 0; |
| 588 |
foreach my $field ($record->field($tag)) { |
| 589 |
$seq++; |
| 590 |
if ($dupes{$seq}) { |
| 591 |
# Here we remove the barcode |
| 592 |
$field->delete_subfield(code => $sub); |
| 593 |
} |
| 594 |
else { |
| 595 |
# otherwise we delete the field because we don't want |
| 596 |
# two of them |
| 597 |
$record->delete_fields($field); |
| 598 |
} |
| 599 |
} |
| 600 |
# Now re-add the record as before, adding errors to the prev list |
| 601 |
my $more_errors; |
| 602 |
eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); }; |
| 603 |
$record_has_added_items ||= @{$itemnumbers_ref}; |
| 604 |
if ($@) { |
| 605 |
warn "ERROR: Adding items to bib $record_id failed: $@\n"; |
| 606 |
printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile); |
| 607 |
# if we failed because of an exception, assume that |
| 608 |
# the MARC columns in biblioitems were not set. |
| 609 |
|
| 610 |
# @FIXME: Why do we save here without stripping items? Besides, |
| 611 |
# save with stripped items has already been performed |
| 612 |
ModBiblioMarc($record, $record_id, $modify_biblio_marc_options); |
| 613 |
next RECORD; |
| 614 |
} |
| 615 |
else { |
| 616 |
printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile); |
| 617 |
} |
| 618 |
push @$errors_ref, @{$more_errors}; |
| 619 |
} |
| 620 |
if (@{$errors_ref}) { |
| 621 |
report_item_errors($record_id, $errors_ref); |
| 622 |
} |
| 623 |
C4::Biblio::_strip_item_fields($record, $framework); |
| 624 |
if ($record_has_added_items || $matched_record_id) { |
| 625 |
# Replace with record from GetMarcBiblio with "$embeditems = 1" |
| 626 |
$record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1}); |
| 627 |
} |
| 628 |
push @search_engine_record_ids, $record_id; |
| 629 |
push @search_engine_records, $record; |
| 543 |
} |
630 |
} |
| 544 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
|
|
| 545 |
} |
631 |
} |
| 546 |
if ( 0 == $i % $commitnum ) { |
632 |
if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) { |
| 547 |
$schema->txn_commit; |
633 |
$schema->txn_commit; |
| 548 |
$schema->txn_begin; |
634 |
$schema->txn_begin; |
|
|
635 |
if ($indexer) { |
| 636 |
$indexer->update_index(\@search_engine_record_ids, \@search_engine_records); |
| 637 |
@search_engine_record_ids = (); |
| 638 |
@search_engine_records = (); |
| 639 |
} |
| 549 |
} |
640 |
} |
| 550 |
} |
641 |
} |
| 551 |
print $record->as_formatted()."\n" if ($verbose//0)==2; |
642 |
print $record->as_formatted() . "\n" if ($verbose//0) == 2; |
| 552 |
last if $i == $number; |
643 |
last if $record_number == $number; |
| 553 |
} |
644 |
} |
| 554 |
$schema->txn_commit; |
645 |
$schema->txn_commit; |
| 555 |
|
646 |
|
| 556 |
if ($fk_off) { |
647 |
if ($fk_off) { |
| 557 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 1"); |
648 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 1"); |
| 558 |
} |
649 |
} |
| 559 |
|
650 |
|
| 560 |
# Restore CataloguingLog and AuthoritiesLog |
651 |
# Restore CataloguingLog and AuthoritiesLog |
|
Lines 562-572
delete $ENV{OVERRIDE_SYSPREF_CataloguingLog};
Link Here
|
| 562 |
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog}; |
653 |
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog}; |
| 563 |
|
654 |
|
| 564 |
my $timeneeded = gettimeofday - $starttime; |
655 |
my $timeneeded = gettimeofday - $starttime; |
| 565 |
print "\n$i MARC records done in $timeneeded seconds\n"; |
656 |
print "\n$record_number MARC records done in $timeneeded seconds\n"; |
| 566 |
if ($logfile){ |
657 |
if ($logfile) { |
| 567 |
print $loghandle "file : $input_marc_file\n"; |
658 |
print $loghandle "file : $input_marc_file\n"; |
| 568 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
659 |
print $loghandle "$record_number MARC records done in $timeneeded seconds\n"; |
| 569 |
$loghandle->close; |
660 |
$loghandle->close; |
| 570 |
} |
661 |
} |
| 571 |
if ($yamlfile) { |
662 |
if ($yamlfile) { |
| 572 |
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; |
663 |
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; |
|
Lines 574-620
if ($yamlfile) {
Link Here
|
| 574 |
} |
665 |
} |
| 575 |
exit 0; |
666 |
exit 0; |
| 576 |
|
667 |
|
| 577 |
sub GetRecordId{ |
668 |
sub YAMLFileEntry { |
| 578 |
my $marcrecord=shift; |
669 |
my ($record, $record_id, $updated) = @_; |
| 579 |
my $tag=shift; |
670 |
|
| 580 |
my $subfield=shift; |
671 |
my $entry = { |
| 581 |
my $id; |
672 |
authid => $record_id |
| 582 |
if ($tag lt "010"){ |
673 |
}; |
| 583 |
return $marcrecord->field($tag)->data() if $marcrecord->field($tag); |
674 |
|
| 584 |
} |
675 |
# we recover all subfields of the heading authorities |
| 585 |
elsif ($subfield){ |
676 |
my @subfields; |
| 586 |
if ($marcrecord->field($tag)){ |
677 |
foreach my $field ($record->field("2..")) { |
| 587 |
return $marcrecord->subfield($tag,$subfield); |
678 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
| 588 |
} |
679 |
} |
| 589 |
} |
680 |
$entry->{'subfields'} = \@subfields; |
| 590 |
return $id; |
681 |
$entry->{'updated'} = $updated; |
|
|
682 |
|
| 683 |
return $entry; |
| 591 |
} |
684 |
} |
| 592 |
sub build_query { |
685 |
|
| 593 |
my $match = shift; |
686 |
sub RecordRevisionIsGtOrEq { |
| 594 |
my $record=shift; |
687 |
my ($record_a, $record_b) = @_; |
| 595 |
my @searchstrings; |
688 |
return $record_a->field('005') && $record_b->field('005') && |
| 596 |
foreach my $matchingpoint (@$match){ |
689 |
$record_a->field('005')->data && $record_b->field('005')->data && |
| 597 |
my $string = build_simplequery($matchingpoint,$record); |
690 |
$record_a->field('005')->data >= $record_b->field('005')->data; |
| 598 |
push @searchstrings,$string if (length($string)>0); |
691 |
} |
|
|
692 |
|
| 693 |
sub GetRecordId { |
| 694 |
my $marcrecord = shift; |
| 695 |
my $tag = shift; |
| 696 |
my $subfield = shift; |
| 697 |
if ($tag lt "010") { |
| 698 |
return $marcrecord->field($tag)->data() if $marcrecord->field($tag); |
| 699 |
} |
| 700 |
elsif ($subfield) { |
| 701 |
if ($marcrecord->field($tag)) { |
| 702 |
return $marcrecord->subfield($tag, $subfield); |
| 599 |
} |
703 |
} |
|
|
704 |
} |
| 705 |
return undef; |
| 706 |
} |
| 707 |
sub build_query { |
| 708 |
my ($match, $record) = @_; |
| 709 |
my @searchstrings; |
| 710 |
|
| 711 |
foreach my $matchpoint (@$match) { |
| 712 |
my $query = build_simplequery($matchpoint, $record); |
| 713 |
push (@searchstrings, $query) if $query; |
| 714 |
} |
| 600 |
my $op = 'and'; |
715 |
my $op = 'and'; |
| 601 |
return join(" $op ",@searchstrings); |
716 |
return join(" $op ", @searchstrings); |
| 602 |
} |
717 |
} |
| 603 |
sub build_simplequery { |
718 |
sub build_simplequery { |
| 604 |
my $element=shift; |
719 |
my ($matchpoint, $record) = @_; |
| 605 |
my $record=shift; |
720 |
|
| 606 |
my @searchstrings; |
721 |
my @searchstrings; |
| 607 |
my ($index,$recorddata)=split /,/,$element; |
722 |
my ($index, $record_data) = split (/,/, $matchpoint); |
| 608 |
if ($recorddata=~/(\d{3})(.*)/) { |
723 |
if ($record_data =~ /(\d{3})(.*)/) { |
| 609 |
my ($tag,$subfields) =($1,$2); |
724 |
my ($tag, $subfields) = ($1, $2); |
| 610 |
foreach my $field ($record->field($tag)){ |
725 |
foreach my $field ($record->field($tag)) { |
| 611 |
if (length($field->as_string("$subfields"))>0){ |
726 |
if (length($field->as_string("$subfields")) > 0) { |
| 612 |
push @searchstrings,"$index:\"".$field->as_string("$subfields")."\""; |
727 |
push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\""); |
| 613 |
} |
728 |
} |
| 614 |
} |
729 |
} |
| 615 |
} |
730 |
} |
|
|
731 |
else { |
| 732 |
print "Invalid matchpoint format, invalid marc-field: $matchpoint\n"; |
| 733 |
} |
| 616 |
my $op = 'and'; |
734 |
my $op = 'and'; |
| 617 |
return join(" $op ",@searchstrings); |
735 |
return join(" $op ", @searchstrings); |
| 618 |
} |
736 |
} |
| 619 |
sub report_item_errors { |
737 |
sub report_item_errors { |
| 620 |
my $biblionumber = shift; |
738 |
my $biblionumber = shift; |
|
Lines 629-648
sub report_item_errors {
Link Here
|
| 629 |
print $msg, "\n"; |
747 |
print $msg, "\n"; |
| 630 |
} |
748 |
} |
| 631 |
} |
749 |
} |
| 632 |
sub printlog{ |
750 |
sub printlog { |
| 633 |
my $logelements=shift; |
751 |
my $logelements = shift; |
| 634 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
752 |
print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n"; |
| 635 |
} |
753 |
} |
| 636 |
sub get_heading_fields{ |
754 |
sub get_heading_fields { |
| 637 |
my $headingfields; |
755 |
my $headingfields; |
| 638 |
if ($authtypes){ |
756 |
if ($authtypes) { |
| 639 |
$headingfields = YAML::XS::LoadFile($authtypes); |
757 |
$headingfields = YAML::XS::LoadFile($authtypes); |
| 640 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
758 |
$headingfields = { C4::Context->preference('marcflavour') => $headingfields }; |
| 641 |
$logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields))); |
759 |
$logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields))); |
| 642 |
} |
760 |
} |
| 643 |
unless ($headingfields){ |
761 |
unless ($headingfields) { |
| 644 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
762 |
$headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
| 645 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
763 |
$headingfields = { C4::Context->preference('marcflavour') => $headingfields }; |
| 646 |
} |
764 |
} |
| 647 |
return $headingfields; |
765 |
return $headingfields; |
| 648 |
} |
766 |
} |
| 649 |
- |
|
|