|
Lines 17-22
use C4::Biblio qw(
Link Here
|
| 17 |
GetMarcFromKohaField |
17 |
GetMarcFromKohaField |
| 18 |
ModBiblio |
18 |
ModBiblio |
| 19 |
ModBiblioMarc |
19 |
ModBiblioMarc |
|
|
20 |
GetFrameworkCode |
| 21 |
GetMarcBiblio |
| 20 |
); |
22 |
); |
| 21 |
use C4::Koha; |
23 |
use C4::Koha; |
| 22 |
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); |
24 |
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag ); |
|
Lines 40-61
use Koha::SearchEngine;
Link Here
|
| 40 |
use Koha::SearchEngine::Search; |
42 |
use Koha::SearchEngine::Search; |
| 41 |
|
43 |
|
| 42 |
use open qw( :std :encoding(UTF-8) ); |
44 |
use open qw( :std :encoding(UTF-8) ); |
| 43 |
binmode( STDOUT, ":encoding(UTF-8)" ); |
45 |
binmode(STDOUT, ":encoding(UTF-8)"); |
| 44 |
my ( $input_marc_file, $number, $offset) = ('',0,0); |
46 |
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1); |
| 45 |
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile); |
47 |
my $version; |
| 46 |
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append ); |
48 |
my $delete; |
| 47 |
my $cleanisbn = 1; |
49 |
my $test_parameter; |
| 48 |
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode); |
50 |
my $skip_marc8_conversion; |
|
|
51 |
my $char_encoding; |
| 52 |
my $verbose; |
| 53 |
my $commit; |
| 54 |
my $fk_off; |
| 55 |
my $format; |
| 56 |
my $biblios; |
| 57 |
my $authorities; |
| 58 |
my $keepids; |
| 59 |
my $match; |
| 60 |
my $isbn_check; |
| 61 |
my $logfile; |
| 62 |
my $insert; |
| 63 |
my $filters; |
| 64 |
my $update; |
| 65 |
my $all; |
| 66 |
my $yamlfile; |
| 67 |
my $authtypes; |
| 68 |
my $append; |
| 69 |
my $sourcetag; |
| 70 |
my $sourcesubfield; |
| 71 |
my $idmapfl; |
| 72 |
my $dedup_barcode; |
| 49 |
my $framework = ''; |
73 |
my $framework = ''; |
| 50 |
my $localcust; |
74 |
my $localcust; |
| 51 |
my $marc_mod_template = ''; |
75 |
my $marc_mod_template = ''; |
| 52 |
my $marc_mod_template_id = -1; |
76 |
my $marc_mod_template_id = -1; |
| 53 |
|
77 |
$| = 1; |
| 54 |
$|=1; |
|
|
| 55 |
|
78 |
|
| 56 |
GetOptions( |
79 |
GetOptions( |
| 57 |
'commit:f' => \$commit, |
80 |
'commit:f' => \$commit, |
| 58 |
'file:s' => \$input_marc_file, |
81 |
'file:s' => \$input_marc_file, |
| 59 |
'n:f' => \$number, |
82 |
'n:f' => \$number, |
| 60 |
'o|offset:f' => \$offset, |
83 |
'o|offset:f' => \$offset, |
| 61 |
'h' => \$version, |
84 |
'h' => \$version, |
|
Lines 72-93
GetOptions(
Link Here
|
| 72 |
'b|biblios' => \$biblios, |
95 |
'b|biblios' => \$biblios, |
| 73 |
'a|authorities' => \$authorities, |
96 |
'a|authorities' => \$authorities, |
| 74 |
'authtypes:s' => \$authtypes, |
97 |
'authtypes:s' => \$authtypes, |
| 75 |
'filter=s@' => \$filters, |
98 |
'filter=s@' => \$filters, |
| 76 |
'insert' => \$insert, |
99 |
'insert' => \$insert, |
| 77 |
'update' => \$update, |
100 |
'update' => \$update, |
| 78 |
'all' => \$all, |
101 |
'all' => \$all, |
| 79 |
'match=s@' => \$match, |
102 |
'match=s@' => \$match, |
| 80 |
'i|isbn' => \$isbn_check, |
103 |
'i|isbn' => \$isbn_check, |
| 81 |
'x:s' => \$sourcetag, |
104 |
'x:s' => \$sourcetag, |
| 82 |
'y:s' => \$sourcesubfield, |
105 |
'y:s' => \$sourcesubfield, |
| 83 |
'idmap:s' => \$idmapfl, |
106 |
'idmap:s' => \$idmapfl, |
| 84 |
'cleanisbn!' => \$cleanisbn, |
107 |
'cleanisbn!' => \$cleanisbn, |
| 85 |
'yaml:s' => \$yamlfile, |
108 |
'yaml:s' => \$yamlfile, |
| 86 |
'dedupbarcode' => \$dedup_barcode, |
109 |
'dedupbarcode' => \$dedup_barcode, |
| 87 |
'framework=s' => \$framework, |
110 |
'framework=s' => \$framework, |
| 88 |
'custom:s' => \$localcust, |
111 |
'custom:s' => \$localcust, |
| 89 |
'marcmodtemplate:s' => \$marc_mod_template, |
112 |
'marcmodtemplate:s' => \$marc_mod_template, |
| 90 |
); |
113 |
); |
|
|
114 |
|
| 91 |
$biblios ||= !$authorities; |
115 |
$biblios ||= !$authorities; |
| 92 |
$insert ||= !$update; |
116 |
$insert ||= !$update; |
| 93 |
my $writemode = ($append) ? "a" : "w"; |
117 |
my $writemode = ($append) ? "a" : "w"; |
|
Lines 99-104
if ($all) {
Link Here
|
| 99 |
$update = 1; |
123 |
$update = 1; |
| 100 |
} |
124 |
} |
| 101 |
|
125 |
|
|
|
126 |
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch'); |
| 127 |
my $modify_biblio_marc_options = { |
| 128 |
defer_search_engine_indexing => $using_elastic_search, |
| 129 |
overlay_context => { source => 'bulkmarcimport' } |
| 130 |
}; |
| 131 |
|
| 132 |
my @search_engine_record_ids; |
| 133 |
my @search_engine_records; |
| 134 |
my $indexer; |
| 135 |
if ($using_elastic_search) { |
| 136 |
use Koha::SearchEngine::Elasticsearch::Indexer; |
| 137 |
$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( |
| 138 |
{ index => $authorities ? |
| 139 |
$Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX : |
| 140 |
$Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX |
| 141 |
} |
| 142 |
); |
| 143 |
} |
| 144 |
|
| 102 |
if ($version || ($input_marc_file eq '')) { |
145 |
if ($version || ($input_marc_file eq '')) { |
| 103 |
pod2usage( -verbose => 2 ); |
146 |
pod2usage( -verbose => 2 ); |
| 104 |
exit; |
147 |
exit; |
|
Lines 107-119
if( $update && !( $match || $isbn_check ) ) {
Link Here
|
| 107 |
warn "Using -update without -match or -isbn seems to be useless.\n"; |
150 |
warn "Using -update without -match or -isbn seems to be useless.\n"; |
| 108 |
} |
151 |
} |
| 109 |
|
152 |
|
| 110 |
if(defined $localcust) { #local customize module |
153 |
if (defined $localcust) { #local customize module |
| 111 |
if(!-e $localcust) { |
154 |
if (!-e $localcust) { |
| 112 |
$localcust= $localcust||'LocalChanges'; #default name |
155 |
$localcust = $localcust || 'LocalChanges'; #default name |
| 113 |
$localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only |
156 |
$localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only |
| 114 |
$localcust=~ s/\.pm$//; #remove extension |
157 |
$localcust =~ s/\.pm$//; #remove extension |
| 115 |
my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir |
158 |
my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir |
| 116 |
if(-e $fqcust) { |
159 |
if (-e $fqcust) { |
| 117 |
$localcust= $fqcust; |
160 |
$localcust= $fqcust; |
| 118 |
} |
161 |
} |
| 119 |
else { |
162 |
else { |
|
Lines 122-128
if(defined $localcust) { #local customize module
Link Here
|
| 122 |
} |
165 |
} |
| 123 |
} |
166 |
} |
| 124 |
require $localcust if $localcust; |
167 |
require $localcust if $localcust; |
| 125 |
$localcust=\&customize if $localcust; |
168 |
$localcust = \&customize if $localcust; |
| 126 |
} |
169 |
} |
| 127 |
|
170 |
|
| 128 |
if($marc_mod_template ne '') { |
171 |
if($marc_mod_template ne '') { |
|
Lines 147-214
if($marc_mod_template ne '') {
Link Here
|
| 147 |
} |
190 |
} |
| 148 |
|
191 |
|
| 149 |
my $dbh = C4::Context->dbh; |
192 |
my $dbh = C4::Context->dbh; |
| 150 |
my $heading_fields=get_heading_fields(); |
193 |
my $heading_fields = get_heading_fields(); |
| 151 |
|
|
|
| 152 |
my $idmapfh; |
194 |
my $idmapfh; |
|
|
195 |
|
| 153 |
if (defined $idmapfl) { |
196 |
if (defined $idmapfl) { |
| 154 |
open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n"; |
197 |
open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n"; |
| 155 |
} |
198 |
} |
| 156 |
|
199 |
|
| 157 |
if ((not defined $sourcesubfield) && (not defined $sourcetag)){ |
200 |
if ((not defined $sourcesubfield) && (not defined $sourcetag)) { |
| 158 |
$sourcetag="910"; |
201 |
$sourcetag = "910"; |
| 159 |
$sourcesubfield="a"; |
202 |
$sourcesubfield = "a"; |
| 160 |
} |
203 |
} |
| 161 |
|
204 |
|
| 162 |
|
|
|
| 163 |
# Disable logging for the biblios and authorities import operation. It would unnecessarily |
205 |
# Disable logging for the biblios and authorities import operation. It would unnecessarily |
| 164 |
# slow the import |
206 |
# slow the import |
| 165 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
207 |
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0; |
| 166 |
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0; |
208 |
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0; |
| 167 |
|
209 |
|
| 168 |
if ($fk_off) { |
210 |
if ($fk_off) { |
| 169 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 0"); |
211 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 0"); |
| 170 |
} |
212 |
} |
| 171 |
|
213 |
|
| 172 |
|
|
|
| 173 |
if ($delete) { |
214 |
if ($delete) { |
| 174 |
if ($biblios){ |
215 |
if ($biblios) { |
| 175 |
print "deleting biblios\n"; |
216 |
print "Deleting biblios\n"; |
| 176 |
$dbh->do("DELETE FROM biblio"); |
217 |
$dbh->do("DELETE FROM biblio"); |
| 177 |
$dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1"); |
218 |
$dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1"); |
| 178 |
$dbh->do("DELETE FROM biblioitems"); |
219 |
$dbh->do("DELETE FROM biblioitems"); |
| 179 |
$dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1"); |
220 |
$dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1"); |
| 180 |
$dbh->do("DELETE FROM items"); |
221 |
$dbh->do("DELETE FROM items"); |
| 181 |
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); |
222 |
$dbh->do("ALTER TABLE items AUTO_INCREMENT = 1"); |
| 182 |
} |
223 |
} |
| 183 |
else { |
224 |
else { |
| 184 |
print "deleting authorities\n"; |
225 |
print "Deleting authorities\n"; |
| 185 |
$dbh->do("truncate auth_header"); |
226 |
$dbh->do("truncate auth_header"); |
| 186 |
} |
227 |
} |
| 187 |
$dbh->do("truncate zebraqueue"); |
228 |
$dbh->do("truncate zebraqueue"); |
| 188 |
} |
229 |
} |
| 189 |
|
230 |
|
| 190 |
|
|
|
| 191 |
|
| 192 |
if ($test_parameter) { |
231 |
if ($test_parameter) { |
| 193 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
232 |
print "TESTING MODE ONLY\n DOING NOTHING\n===============\n"; |
| 194 |
} |
233 |
} |
| 195 |
|
234 |
|
| 196 |
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21'; |
235 |
my $batch; |
|
|
236 |
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21'; |
| 197 |
|
237 |
|
| 198 |
# The definition of $searcher must be before MARC::Batch->new |
238 |
# The definition of $searcher must be before MARC::Batch->new |
| 199 |
my $searcher = Koha::SearchEngine::Search->new( |
239 |
my $searcher = Koha::SearchEngine::Search->new( |
| 200 |
{ |
240 |
{ |
| 201 |
index => ( |
241 |
index => ( |
| 202 |
$authorities |
242 |
$authorities |
| 203 |
? $Koha::SearchEngine::AUTHORITIES_INDEX |
243 |
? $Koha::SearchEngine::AUTHORITIES_INDEX |
| 204 |
: $Koha::SearchEngine::BIBLIOS_INDEX |
244 |
: $Koha::SearchEngine::BIBLIOS_INDEX |
| 205 |
) |
245 |
) |
| 206 |
} |
246 |
} |
| 207 |
); |
247 |
); |
| 208 |
|
248 |
|
| 209 |
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose; |
249 |
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose; |
| 210 |
my $starttime = gettimeofday; |
250 |
my $starttime = gettimeofday; |
| 211 |
my $batch; |
251 |
|
| 212 |
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
252 |
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer |
| 213 |
if (defined $format && $format =~ /XML/i) { |
253 |
if (defined $format && $format =~ /XML/i) { |
| 214 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
254 |
# ugly hack follows -- MARC::File::XML, when used by MARC::Batch, |
|
Lines 221-274
if (defined $format && $format =~ /XML/i) {
Link Here
|
| 221 |
# extract the records, not using regexes to look |
261 |
# extract the records, not using regexes to look |
| 222 |
# for <record>.*</record>. |
262 |
# for <record>.*</record>. |
| 223 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
263 |
$MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8'; |
| 224 |
my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour)); |
264 |
my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour)); |
| 225 |
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. |
265 |
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios. |
| 226 |
$recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21"); |
266 |
$recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21"); |
| 227 |
$MARC::File::XML::_load_args{RecordFormat} = $recordformat; |
267 |
$MARC::File::XML::_load_args{RecordFormat} = $recordformat; |
| 228 |
$batch = MARC::Batch->new( 'XML', $fh ); |
268 |
$batch = MARC::Batch->new('XML', $fh); |
| 229 |
} else { |
|
|
| 230 |
$batch = MARC::Batch->new( 'USMARC', $fh ); |
| 231 |
} |
269 |
} |
|
|
270 |
else { |
| 271 |
$batch = MARC::Batch->new('USMARC', $fh); |
| 272 |
} |
| 273 |
|
| 232 |
$batch->warnings_off(); |
274 |
$batch->warnings_off(); |
| 233 |
$batch->strict_off(); |
275 |
$batch->strict_off(); |
| 234 |
my $i=0; |
|
|
| 235 |
my $commitnum = $commit ? $commit : 50; |
276 |
my $commitnum = $commit ? $commit : 50; |
| 236 |
my $yamlhash; |
277 |
my $yamlhash; |
| 237 |
|
278 |
|
| 238 |
# Skip file offset |
279 |
# Skip file offset |
| 239 |
if ( $offset ) { |
280 |
if ($offset) { |
| 240 |
print "Skipping file offset: $offset records\n"; |
281 |
print "Skipping file offset: $offset records\n"; |
| 241 |
$batch->next() while ($offset--); |
282 |
$batch->next() while ($offset--); |
| 242 |
} |
283 |
} |
| 243 |
|
284 |
|
| 244 |
my ($tagid,$subfieldid); |
285 |
my ($tagid, $subfieldid); |
| 245 |
if ($authorities){ |
286 |
if ($authorities) { |
| 246 |
$tagid='001'; |
287 |
$tagid = '001'; |
| 247 |
} |
288 |
} |
| 248 |
else { |
289 |
else { |
| 249 |
( $tagid, $subfieldid ) = |
290 |
($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber"); |
| 250 |
GetMarcFromKohaField( "biblio.biblionumber" ); |
291 |
$tagid ||= "001"; |
| 251 |
$tagid||="001"; |
|
|
| 252 |
} |
292 |
} |
| 253 |
|
293 |
|
|
|
294 |
my $sth_isbn; |
| 254 |
# the SQL query to search on isbn |
295 |
# the SQL query to search on isbn |
| 255 |
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?"); |
296 |
if ($isbn_check) { |
|
|
297 |
$sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?"); |
| 298 |
} |
| 256 |
|
299 |
|
| 257 |
my $loghandle; |
300 |
my $loghandle; |
| 258 |
if ($logfile){ |
301 |
if ($logfile) { |
| 259 |
$loghandle= IO::File->new($logfile, $writemode) ; |
302 |
$loghandle= IO::File->new($logfile, $writemode); |
| 260 |
print $loghandle "id;operation;status\n"; |
303 |
print $loghandle "id;operation;status\n"; |
| 261 |
} |
304 |
} |
| 262 |
|
305 |
|
|
|
306 |
my $record_number = 0; |
| 263 |
my $logger = Koha::Logger->get; |
307 |
my $logger = Koha::Logger->get; |
| 264 |
my $schema = Koha::Database->schema; |
308 |
my $schema = Koha::Database->schema; |
| 265 |
$schema->txn_begin; |
309 |
my $marc_records = []; |
| 266 |
RECORD: while ( ) { |
310 |
RECORD: while ( ) { |
| 267 |
my $record; |
311 |
my $record; |
| 268 |
# get records |
312 |
$record_number++; |
|
|
313 |
# get record |
| 269 |
eval { $record = $batch->next() }; |
314 |
eval { $record = $batch->next() }; |
| 270 |
if ( $@ ) { |
315 |
if ($@) { |
| 271 |
print "Bad MARC record $i: $@ skipped\n"; |
316 |
print "Bad MARC record $record_number: $@ skipped\n"; |
| 272 |
# FIXME - because MARC::Batch->next() combines grabbing the next |
317 |
# FIXME - because MARC::Batch->next() combines grabbing the next |
| 273 |
# blob and parsing it into one operation, a correctable condition |
318 |
# blob and parsing it into one operation, a correctable condition |
| 274 |
# such as a MARC-8 record claiming that it's UTF-8 can't be recovered |
319 |
# such as a MARC-8 record claiming that it's UTF-8 can't be recovered |
|
Lines 277-555
RECORD: while ( ) {
Link Here
|
| 277 |
# C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. |
322 |
# C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch. |
| 278 |
next; |
323 |
next; |
| 279 |
} |
324 |
} |
| 280 |
# skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure |
325 |
if ($record) { |
| 281 |
last unless ( $record ); |
326 |
# transcode the record to UTF8 if needed & applicable. |
| 282 |
$i++; |
327 |
if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { |
| 283 |
if( ($verbose//1)==1 ) { #no dot for verbose==2 |
328 |
my ($guessed_charset, $charset_errors); |
| 284 |
print "." . ( $i % 100==0 ? "\n$i" : '' ); |
329 |
($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : '')); |
|
|
330 |
if ($guessed_charset eq 'failed') { |
| 331 |
warn "ERROR: failed to perform character conversion for record $record_number\n"; |
| 332 |
next RECORD; |
| 333 |
} |
| 334 |
} |
| 335 |
SetUTF8Flag($record); |
| 336 |
&$localcust($record) if $localcust; |
| 337 |
push @{$marc_records}, $record; |
| 338 |
} |
| 339 |
else { |
| 340 |
last; |
| 285 |
} |
341 |
} |
|
|
342 |
} |
| 286 |
|
343 |
|
| 287 |
# transcode the record to UTF8 if needed & applicable. |
344 |
$record_number = 0; |
| 288 |
if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) { |
345 |
my $records_total = @{$marc_records}; |
| 289 |
# FIXME update condition |
346 |
$schema->txn_begin; |
| 290 |
my ($guessed_charset, $charset_errors); |
347 |
RECORD: foreach my $record (@{$marc_records}) { |
| 291 |
($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':'')); |
348 |
$record_number++; |
| 292 |
if ($guessed_charset eq 'failed') { |
349 |
if (($verbose//1) == 1) { #no dot for verbose==2 |
| 293 |
warn "ERROR: failed to perform character conversion for record $i\n"; |
350 |
print "." . ($record_number % 100 == 0 ? "\n$record_number" : ''); |
| 294 |
next RECORD; |
|
|
| 295 |
} |
| 296 |
} |
351 |
} |
| 297 |
SetUTF8Flag($record); |
352 |
|
| 298 |
if($marc_mod_template_id > 0) { |
353 |
if ($marc_mod_template_id > 0) { |
| 299 |
print "Modifying MARC\n" if $verbose; |
354 |
print "Modifying MARC\n" if $verbose; |
| 300 |
ModifyRecordWithTemplate( $marc_mod_template_id, $record ); |
355 |
ModifyRecordWithTemplate( $marc_mod_template_id, $record ); |
| 301 |
} |
356 |
} |
| 302 |
&$localcust($record) if $localcust; |
357 |
|
| 303 |
my $isbn; |
358 |
my $isbn; |
| 304 |
# remove trailing - in isbn (only for biblios, of course) |
359 |
# remove trailing - in isbn (only for biblios, of course) |
| 305 |
if( $biblios ) { |
360 |
if ($biblios && ($cleanisbn || $isbn_check)) { |
| 306 |
my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020'; |
361 |
my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020'; |
| 307 |
my $field = $record->field($tag); |
362 |
my $field = $record->field($tag); |
| 308 |
$isbn = $field && $field->subfield('a'); |
363 |
$isbn = $field && $field->subfield('a'); |
| 309 |
if ( $isbn && $cleanisbn ) { |
364 |
if ($isbn && $cleanisbn) { |
| 310 |
$isbn =~ s/-//g; |
365 |
$isbn =~ s/-//g; |
| 311 |
$field->update('a' => $isbn); |
366 |
$field->update('a' => $isbn); |
| 312 |
} |
367 |
} |
| 313 |
} |
368 |
} |
| 314 |
my $id; |
|
|
| 315 |
# search for duplicates (based on Local-number) |
369 |
# search for duplicates (based on Local-number) |
| 316 |
my $originalid; |
370 |
my $originalid = GetRecordId($record, $tagid, $subfieldid); |
| 317 |
$originalid = GetRecordId( $record, $tagid, $subfieldid ); |
371 |
my $matched_record_id = undef; |
| 318 |
if ($match) { |
372 |
if ($match) { |
| 319 |
require C4::Search; |
373 |
require C4::Search; |
| 320 |
my $query = build_query( $match, $record ); |
374 |
my $server = ($authorities ? 'authorityserver' : 'biblioserver'); |
| 321 |
my $server = ( $authorities ? 'authorityserver' : 'biblioserver' ); |
375 |
my $query = build_query($match, $record); |
| 322 |
my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] ); |
376 |
$logger->debug("Bulkmarcimport: $query"); |
|
|
377 |
my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]); |
| 323 |
# changed to warn so able to continue with one broken record |
378 |
# changed to warn so able to continue with one broken record |
| 324 |
if ( defined $error ) { |
379 |
if (defined $error) { |
| 325 |
warn "unable to search the database for duplicates : $error"; |
380 |
warn "unable to search the database for duplicates : $error"; |
| 326 |
printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile); |
381 |
printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile); |
| 327 |
next RECORD; |
382 |
next RECORD; |
| 328 |
} |
383 |
} |
| 329 |
if ( $results && scalar(@$results) == 1 ) { |
384 |
$logger->debug("Bulkmarcimport: $query $server : $totalhits"); |
| 330 |
my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] ); |
385 |
# sub SimpleSearch could return undefined, but only on error, so |
| 331 |
SetUTF8Flag($marcrecord); |
386 |
# should not really need to safeguard here, but do so anyway |
| 332 |
$id = GetRecordId( $marcrecord, $tagid, $subfieldid ); |
387 |
$results //= []; |
| 333 |
if ( $authorities && $marcFlavour ) { |
388 |
if (@{$results} == 1) { |
| 334 |
#Skip if authority in database is the same as the on in database |
389 |
my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]); |
| 335 |
if ( $marcrecord->field('005') && $record->field('005') && |
390 |
SetUTF8Flag($matched_record); |
| 336 |
$marcrecord->field('005')->data && $record->field('005')->data && |
391 |
$matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid); |
| 337 |
$marcrecord->field('005')->data >= $record->field('005')->data ) { |
392 |
|
|
|
393 |
if ($authorities && $marc_flavour) { |
| 394 |
#Skip if authority in database is the same or newer than the incoming record |
| 395 |
if (RecordRevisionIsGtOrEq($matched_record, $record)) { |
| 338 |
if ($yamlfile) { |
396 |
if ($yamlfile) { |
| 339 |
$yamlhash->{$originalid}->{'authid'} = $id; |
397 |
$yamlhash->{$originalid} = YAMLFileEntry( |
| 340 |
|
398 |
$matched_record, |
| 341 |
# we recover all subfields of the heading authorities |
399 |
$matched_record_id, |
| 342 |
my @subfields; |
400 |
0 |
| 343 |
foreach my $field ( $marcrecord->field("2..") ) { |
401 |
); |
| 344 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
|
|
| 345 |
} |
| 346 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
| 347 |
$yamlhash->{$originalid}->{'updated'} = 0; |
| 348 |
} |
402 |
} |
| 349 |
next; |
403 |
next; |
| 350 |
} |
404 |
} |
| 351 |
} |
405 |
} |
| 352 |
} elsif ( $results && scalar(@$results) > 1 ) { |
|
|
| 353 |
$logger->debug("more than one match for $query"); |
| 354 |
} else { |
| 355 |
$logger->debug("nomatch for $query"); |
| 356 |
} |
406 |
} |
| 357 |
} |
407 |
elsif(@{$results} > 1) { |
| 358 |
if ($keepids && $originalid) { |
408 |
$logger->debug("More than one match for: $query"); |
|
|
409 |
} |
| 410 |
else { |
| 411 |
$logger->debug("No match for: $query"); |
| 412 |
} |
| 413 |
|
| 414 |
if ($keepids && $originalid) { |
| 359 |
my $storeidfield; |
415 |
my $storeidfield; |
| 360 |
if ( length($keepids) == 3 ) { |
416 |
if (length($keepids) == 3) { |
| 361 |
$storeidfield = MARC::Field->new( $keepids, $originalid ); |
417 |
$storeidfield = MARC::Field->new($keepids, $originalid); |
| 362 |
} else { |
418 |
} else { |
| 363 |
$storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid ); |
419 |
$storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid); |
| 364 |
} |
420 |
} |
| 365 |
$record->insert_fields_ordered($storeidfield); |
421 |
$record->insert_fields_ordered($storeidfield); |
| 366 |
$record->delete_field( $record->field($tagid) ); |
422 |
$record->delete_field($record->field($tagid)); |
|
|
423 |
} |
| 367 |
} |
424 |
} |
|
|
425 |
|
| 368 |
foreach my $stringfilter (@$filters) { |
426 |
foreach my $stringfilter (@$filters) { |
| 369 |
if ( length($stringfilter) == 3 ) { |
427 |
if (length($stringfilter) == 3) { |
| 370 |
foreach my $field ( $record->field($stringfilter) ) { |
428 |
foreach my $field ($record->field($stringfilter)) { |
| 371 |
$record->delete_field($field); |
429 |
$record->delete_field($field); |
| 372 |
$logger->debug("removed : ", $field->as_string); |
430 |
$logger->debug("Removed: ", $field->as_string); |
| 373 |
} |
431 |
} |
| 374 |
} elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { |
432 |
} elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) { |
| 375 |
my $removetag = $1; |
433 |
my $removetag = $1; |
| 376 |
my $removesubfield = $2; |
434 |
my $removesubfield = $2; |
| 377 |
my $removematch = $3; |
435 |
my $removematch = $3; |
| 378 |
if ( ( $removetag > "010" ) && $removesubfield ) { |
436 |
if (($removetag > "010") && $removesubfield) { |
| 379 |
foreach my $field ( $record->field($removetag) ) { |
437 |
foreach my $field ($record->field($removetag)) { |
| 380 |
$field->delete_subfield( code => "$removesubfield", match => $removematch ); |
438 |
$field->delete_subfield(code => "$removesubfield", match => $removematch); |
| 381 |
$logger->debug("Potentially removed : ", $field->subfield($removesubfield)); |
439 |
$logger->debug("Potentially removed: ", $field->subfield($removesubfield)); |
| 382 |
} |
440 |
} |
| 383 |
} |
441 |
} |
| 384 |
} |
442 |
} |
| 385 |
} |
443 |
} |
| 386 |
unless ($test_parameter) { |
444 |
unless ($test_parameter) { |
| 387 |
if ($authorities){ |
445 |
if ($authorities) { |
| 388 |
my $authtypecode=GuessAuthTypeCode($record, $heading_fields); |
446 |
my $authtypecode = GuessAuthTypeCode($record, $heading_fields); |
| 389 |
my $authid= ($id?$id:GuessAuthId($record)); |
447 |
my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record)); |
| 390 |
if ($authid && GetAuthority($authid) && $update ){ |
448 |
if ($authid && GetAuthority($authid) && $update) { |
| 391 |
## Authority has an id and is in database : Replace |
449 |
## Authority has an id and is in database : Replace |
| 392 |
eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) }; |
450 |
eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) }; |
| 393 |
if ($@){ |
451 |
if ($@) { |
| 394 |
warn "Problem with authority $authid Cannot Modify"; |
452 |
warn "Problem with authority $authid Cannot Modify"; |
| 395 |
printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile); |
453 |
printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile); |
|
|
454 |
} |
| 455 |
else{ |
| 456 |
printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile); |
| 396 |
} |
457 |
} |
| 397 |
else{ |
458 |
} |
| 398 |
printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile); |
459 |
else { |
| 399 |
} |
|
|
| 400 |
} |
| 401 |
else { |
| 402 |
## True insert in database |
460 |
## True insert in database |
| 403 |
eval { ( $authid ) = AddAuthority($record,"", $authtypecode) }; |
461 |
eval { ( $authid ) = AddAuthority($record, "", $authtypecode) }; |
| 404 |
if ($@){ |
462 |
if ($@) { |
| 405 |
warn "Problem with authority $authid Cannot Add".$@; |
463 |
warn "Problem with authority $originalid Cannot Add ".$@; |
| 406 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile); |
464 |
printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile); |
| 407 |
} |
465 |
} |
| 408 |
else{ |
466 |
else { |
| 409 |
printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile); |
467 |
printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile); |
| 410 |
} |
468 |
} |
| 411 |
} |
469 |
|
| 412 |
if ($yamlfile) { |
|
|
| 413 |
$yamlhash->{$originalid}->{'authid'} = $authid; |
| 414 |
my @subfields; |
| 415 |
foreach my $field ( $record->field("2..") ) { |
| 416 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
| 417 |
} |
470 |
} |
| 418 |
$yamlhash->{$originalid}->{'subfields'} = \@subfields; |
471 |
if ($yamlfile) { |
| 419 |
$yamlhash->{$originalid}->{'updated'} = 1; |
472 |
$yamlhash->{$originalid} = YAMLFileEntry( |
|
|
473 |
$record, |
| 474 |
$authid, |
| 475 |
1 #@FIXME: Really always updated? |
| 476 |
); |
| 420 |
} |
477 |
} |
| 421 |
} |
478 |
} |
| 422 |
else { |
479 |
else { |
| 423 |
my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref ); |
480 |
my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id); |
| 424 |
$biblionumber = $id; |
|
|
| 425 |
# check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter |
481 |
# check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter |
| 426 |
if (!$biblionumber && $isbn_check && $isbn) { |
482 |
if (!$matched_record_id && $isbn_check && $isbn) { |
| 427 |
# warn "search ISBN : $isbn"; |
|
|
| 428 |
$sth_isbn->execute($isbn); |
483 |
$sth_isbn->execute($isbn); |
| 429 |
($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow; |
484 |
($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow; |
| 430 |
} |
485 |
} |
| 431 |
if (defined $idmapfl) { |
486 |
|
| 432 |
if ($sourcetag < "010"){ |
487 |
if (defined $idmapfl && $matched_record_id) { |
| 433 |
if ($record->field($sourcetag)){ |
488 |
if ($sourcetag < "010") { |
| 434 |
my $source = $record->field($sourcetag)->data(); |
489 |
if ($record->field($sourcetag)) { |
| 435 |
printf($idmapfh "%s|%s\n",$source,$biblionumber); |
490 |
my $source = $record->field($sourcetag)->data(); |
| 436 |
} |
491 |
printf($idmapfh "%s|%s\n", $source, $matched_record_id); |
| 437 |
} else { |
492 |
} |
| 438 |
my $source=$record->subfield($sourcetag,$sourcesubfield); |
493 |
} |
| 439 |
printf($idmapfh "%s|%s\n",$source,$biblionumber); |
494 |
else { |
| 440 |
} |
495 |
my $source = $record->subfield($sourcetag, $sourcesubfield); |
| 441 |
} |
496 |
printf($idmapfh "%s|%s\n", $source, $matched_record_id); |
| 442 |
# create biblio, unless we already have it ( either match or isbn ) |
497 |
} |
| 443 |
if ($biblionumber) { |
498 |
} |
|
|
499 |
|
| 500 |
# create biblio, unless we already have it ( either match or isbn ) |
| 501 |
if ($matched_record_id) { |
| 502 |
# TODO: Implement also for authority records! |
| 444 |
eval{ |
503 |
eval{ |
| 445 |
$biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber; |
504 |
$biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber; |
| 446 |
}; |
505 |
}; |
| 447 |
if ($update) { |
506 |
if ($update) { |
| 448 |
eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) }; |
507 |
my $success; |
|
|
508 |
eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) }; |
| 449 |
if ($@) { |
509 |
if ($@) { |
| 450 |
warn "ERROR: Edit biblio $biblionumber failed: $@\n"; |
510 |
warn "ERROR: Edit biblio $matched_record_id failed: $@\n"; |
| 451 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); |
511 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
| 452 |
next RECORD; |
512 |
next RECORD; |
| 453 |
} else { |
|
|
| 454 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile); |
| 455 |
} |
513 |
} |
| 456 |
} else { |
514 |
elsif (!$success) { |
| 457 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile); |
515 |
warn "ERROR: Edit biblio $matched_record_id failed for unkown reason"; |
| 458 |
} |
516 |
printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile); |
| 459 |
} else { |
|
|
| 460 |
if ($insert) { |
| 461 |
eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) }; |
| 462 |
if ($@) { |
| 463 |
warn "ERROR: Adding biblio $biblionumber failed: $@\n"; |
| 464 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile); |
| 465 |
next RECORD; |
517 |
next RECORD; |
| 466 |
} else { |
|
|
| 467 |
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile); |
| 468 |
} |
518 |
} |
| 469 |
} else { |
519 |
else { |
| 470 |
warn "WARNING: Updating record ".($id||$originalid)." failed"; |
520 |
$record_id = $matched_record_id; |
| 471 |
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile); |
521 |
printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile); |
|
|
522 |
} |
| 523 |
} |
| 524 |
else { |
| 525 |
printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile); |
| 526 |
} |
| 527 |
} |
| 528 |
elsif ($insert) { |
| 529 |
eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) }; |
| 530 |
if ($@) { |
| 531 |
warn "ERROR: Adding biblio $record_id failed: $@\n"; |
| 532 |
printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile); |
| 472 |
next RECORD; |
533 |
next RECORD; |
| 473 |
} |
534 |
} |
|
|
535 |
else { |
| 536 |
printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile); |
| 537 |
} |
| 474 |
} |
538 |
} |
| 475 |
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
539 |
else { |
| 476 |
my $error_adding = $@; |
540 |
warn "WARNING: Updating record ".($originalid)." failed"; |
| 477 |
# Work on a clone so that if there are real errors, we can maybe |
541 |
printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile); |
| 478 |
# fix them up later. |
|
|
| 479 |
my $clone_record = $record->clone(); |
| 480 |
C4::Biblio::_strip_item_fields($clone_record, ''); |
| 481 |
# This sets the marc fields if there was an error, and also calls |
| 482 |
# defer_marc_save. |
| 483 |
ModBiblioMarc( $clone_record, $biblionumber ); |
| 484 |
if ( $error_adding ) { |
| 485 |
warn "ERROR: Adding items to bib $biblionumber failed: $error_adding"; |
| 486 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile); |
| 487 |
# if we failed because of an exception, assume that |
| 488 |
# the MARC columns in biblioitems were not set. |
| 489 |
next RECORD; |
542 |
next RECORD; |
| 490 |
} |
543 |
} |
| 491 |
else{ |
544 |
my $record_has_added_items = 0; |
| 492 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile); |
545 |
if ($record_id) { |
| 493 |
} |
546 |
$yamlhash->{$originalid} = $record_id if $yamlfile; |
| 494 |
if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) { |
547 |
eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); }; |
| 495 |
# Find the record called 'barcode' |
548 |
$record_has_added_items = @{$itemnumbers_ref}; |
| 496 |
my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' ); |
549 |
my $error_adding = $@; |
| 497 |
# Now remove any items that didn't have a duplicate_barcode error, |
550 |
# Work on a clone so that if there are real errors, we can maybe |
| 498 |
# erase the barcodes on items that did, and re-add those items. |
551 |
# fix them up later. |
| 499 |
my %dupes; |
552 |
my $clone_record = $record->clone(); |
| 500 |
foreach my $i (0 .. $#{$errors_ref}) { |
553 |
C4::Biblio::_strip_item_fields($clone_record, $framework); |
| 501 |
my $ref = $errors_ref->[$i]; |
554 |
# This sets the marc fields if there was an error, and also calls |
| 502 |
if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) { |
555 |
# defer_marc_save. |
| 503 |
$dupes{$ref->{item_sequence}} = 1; |
556 |
ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options); |
| 504 |
# Delete the error message because we're going to |
557 |
if ($error_adding) { |
| 505 |
# retry this one. |
558 |
warn "ERROR: Adding items to bib $record_id failed: $error_adding"; |
| 506 |
delete $errors_ref->[$i]; |
559 |
printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile); |
| 507 |
} |
|
|
| 508 |
} |
| 509 |
my $seq = 0; |
| 510 |
foreach my $field ($record->field($tag)) { |
| 511 |
$seq++; |
| 512 |
if ($dupes{$seq}) { |
| 513 |
# Here we remove the barcode |
| 514 |
$field->delete_subfield(code => $sub); |
| 515 |
} else { |
| 516 |
# otherwise we delete the field because we don't want |
| 517 |
# two of them |
| 518 |
$record->delete_fields($field); |
| 519 |
} |
| 520 |
} |
| 521 |
# Now re-add the record as before, adding errors to the prev list |
| 522 |
my $more_errors; |
| 523 |
eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); }; |
| 524 |
if ( $@ ) { |
| 525 |
warn "ERROR: Adding items to bib $biblionumber failed: $@\n"; |
| 526 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile); |
| 527 |
# if we failed because of an exception, assume that |
560 |
# if we failed because of an exception, assume that |
| 528 |
# the MARC columns in biblioitems were not set. |
561 |
# the MARC columns in biblioitems were not set. |
| 529 |
ModBiblioMarc( $record, $biblionumber ); |
|
|
| 530 |
next RECORD; |
562 |
next RECORD; |
| 531 |
} else { |
|
|
| 532 |
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile); |
| 533 |
} |
563 |
} |
| 534 |
push @$errors_ref, @{ $more_errors }; |
564 |
else { |
| 535 |
} |
565 |
printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile); |
| 536 |
if ($#{ $errors_ref } > -1) { |
566 |
} |
| 537 |
report_item_errors($biblionumber, $errors_ref); |
567 |
if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) { |
|
|
568 |
# Find the record called 'barcode' |
| 569 |
my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode'); |
| 570 |
# Now remove any items that didn't have a duplicate_barcode error, |
| 571 |
# erase the barcodes on items that did, and re-add those items. |
| 572 |
my %dupes; |
| 573 |
foreach my $i (0 .. $#{$errors_ref}) { |
| 574 |
my $ref = $errors_ref->[$i]; |
| 575 |
if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) { |
| 576 |
$dupes{$ref->{item_sequence}} = 1; |
| 577 |
# Delete the error message because we're going to |
| 578 |
# retry this one. |
| 579 |
delete $errors_ref->[$i]; |
| 580 |
} |
| 581 |
} |
| 582 |
my $seq = 0; |
| 583 |
foreach my $field ($record->field($tag)) { |
| 584 |
$seq++; |
| 585 |
if ($dupes{$seq}) { |
| 586 |
# Here we remove the barcode |
| 587 |
$field->delete_subfield(code => $sub); |
| 588 |
} |
| 589 |
else { |
| 590 |
# otherwise we delete the field because we don't want |
| 591 |
# two of them |
| 592 |
$record->delete_fields($field); |
| 593 |
} |
| 594 |
} |
| 595 |
# Now re-add the record as before, adding errors to the prev list |
| 596 |
my $more_errors; |
| 597 |
eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); }; |
| 598 |
$record_has_added_items ||= @{$itemnumbers_ref}; |
| 599 |
if ($@) { |
| 600 |
warn "ERROR: Adding items to bib $record_id failed: $@\n"; |
| 601 |
printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile); |
| 602 |
# if we failed because of an exception, assume that |
| 603 |
# the MARC columns in biblioitems were not set. |
| 604 |
|
| 605 |
# @FIXME: Why do we save here without stripping items? Besides, |
| 606 |
# save with stripped items has already been performed |
| 607 |
ModBiblioMarc($record, $record_id, $modify_biblio_marc_options); |
| 608 |
next RECORD; |
| 609 |
} |
| 610 |
else { |
| 611 |
printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile); |
| 612 |
} |
| 613 |
push @$errors_ref, @{$more_errors}; |
| 614 |
} |
| 615 |
if (@{$errors_ref}) { |
| 616 |
report_item_errors($record_id, $errors_ref); |
| 617 |
} |
| 618 |
C4::Biblio::_strip_item_fields($record, $framework); |
| 619 |
if ($record_has_added_items || $matched_record_id) { |
| 620 |
# Replace with record from GetMarcBiblio with "$embeditems = 1" |
| 621 |
$record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1}); |
| 622 |
} |
| 623 |
push @search_engine_record_ids, $record_id; |
| 624 |
push @search_engine_records, $record; |
| 538 |
} |
625 |
} |
| 539 |
$yamlhash->{$originalid} = $biblionumber if ($yamlfile); |
|
|
| 540 |
} |
626 |
} |
| 541 |
if ( 0 == $i % $commitnum ) { |
627 |
if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) { |
| 542 |
$schema->txn_commit; |
628 |
$schema->txn_commit; |
| 543 |
$schema->txn_begin; |
629 |
$schema->txn_begin; |
|
|
630 |
if ($indexer) { |
| 631 |
$indexer->update_index(\@search_engine_record_ids, \@search_engine_records); |
| 632 |
@search_engine_record_ids = (); |
| 633 |
@search_engine_records = (); |
| 634 |
} |
| 544 |
} |
635 |
} |
| 545 |
} |
636 |
} |
| 546 |
print $record->as_formatted()."\n" if ($verbose//0)==2; |
637 |
print $record->as_formatted() . "\n" if ($verbose//0) == 2; |
| 547 |
last if $i == $number; |
638 |
last if $record_number == $number; |
| 548 |
} |
639 |
} |
| 549 |
$schema->txn_commit; |
640 |
$schema->txn_commit; |
| 550 |
|
641 |
|
| 551 |
if ($fk_off) { |
642 |
if ($fk_off) { |
| 552 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 1"); |
643 |
$dbh->do("SET FOREIGN_KEY_CHECKS = 1"); |
| 553 |
} |
644 |
} |
| 554 |
|
645 |
|
| 555 |
# Restore CataloguingLog and AuthoritiesLog |
646 |
# Restore CataloguingLog and AuthoritiesLog |
|
Lines 557-567
delete $ENV{OVERRIDE_SYSPREF_CataloguingLog};
Link Here
|
| 557 |
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog}; |
648 |
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog}; |
| 558 |
|
649 |
|
| 559 |
my $timeneeded = gettimeofday - $starttime; |
650 |
my $timeneeded = gettimeofday - $starttime; |
| 560 |
print "\n$i MARC records done in $timeneeded seconds\n"; |
651 |
print "\n$record_number MARC records done in $timeneeded seconds\n"; |
| 561 |
if ($logfile){ |
652 |
if ($logfile) { |
| 562 |
print $loghandle "file : $input_marc_file\n"; |
653 |
print $loghandle "file : $input_marc_file\n"; |
| 563 |
print $loghandle "$i MARC records done in $timeneeded seconds\n"; |
654 |
print $loghandle "$record_number MARC records done in $timeneeded seconds\n"; |
| 564 |
$loghandle->close; |
655 |
$loghandle->close; |
| 565 |
} |
656 |
} |
| 566 |
if ($yamlfile) { |
657 |
if ($yamlfile) { |
| 567 |
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; |
658 |
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n"; |
|
Lines 569-615
if ($yamlfile) {
Link Here
|
| 569 |
} |
660 |
} |
| 570 |
exit 0; |
661 |
exit 0; |
| 571 |
|
662 |
|
| 572 |
sub GetRecordId{ |
663 |
sub YAMLFileEntry { |
| 573 |
my $marcrecord=shift; |
664 |
my ($record, $record_id, $updated) = @_; |
| 574 |
my $tag=shift; |
665 |
|
| 575 |
my $subfield=shift; |
666 |
my $entry = { |
| 576 |
my $id; |
667 |
authid => $record_id |
| 577 |
if ($tag lt "010"){ |
668 |
}; |
| 578 |
return $marcrecord->field($tag)->data() if $marcrecord->field($tag); |
669 |
|
| 579 |
} |
670 |
# we recover all subfields of the heading authorities |
| 580 |
elsif ($subfield){ |
671 |
my @subfields; |
| 581 |
if ($marcrecord->field($tag)){ |
672 |
foreach my $field ($record->field("2..")) { |
| 582 |
return $marcrecord->subfield($tag,$subfield); |
673 |
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields(); |
| 583 |
} |
674 |
} |
| 584 |
} |
675 |
$entry->{'subfields'} = \@subfields; |
| 585 |
return $id; |
676 |
$entry->{'updated'} = $updated; |
|
|
677 |
|
| 678 |
return $entry; |
| 586 |
} |
679 |
} |
| 587 |
sub build_query { |
680 |
|
| 588 |
my $match = shift; |
681 |
sub RecordRevisionIsGtOrEq { |
| 589 |
my $record=shift; |
682 |
my ($record_a, $record_b) = @_; |
| 590 |
my @searchstrings; |
683 |
return $record_a->field('005') && $record_b->field('005') && |
| 591 |
foreach my $matchingpoint (@$match){ |
684 |
$record_a->field('005')->data && $record_b->field('005')->data && |
| 592 |
my $string = build_simplequery($matchingpoint,$record); |
685 |
$record_a->field('005')->data >= $record_b->field('005')->data; |
| 593 |
push @searchstrings,$string if (length($string)>0); |
686 |
} |
|
|
687 |
|
| 688 |
sub GetRecordId { |
| 689 |
my $marcrecord = shift; |
| 690 |
my $tag = shift; |
| 691 |
my $subfield = shift; |
| 692 |
if ($tag lt "010") { |
| 693 |
return $marcrecord->field($tag)->data() if $marcrecord->field($tag); |
| 694 |
} |
| 695 |
elsif ($subfield) { |
| 696 |
if ($marcrecord->field($tag)) { |
| 697 |
return $marcrecord->subfield($tag, $subfield); |
| 594 |
} |
698 |
} |
|
|
699 |
} |
| 700 |
return undef; |
| 701 |
} |
| 702 |
sub build_query { |
| 703 |
my ($match, $record) = @_; |
| 704 |
my @searchstrings; |
| 705 |
|
| 706 |
foreach my $matchpoint (@$match) { |
| 707 |
my $query = build_simplequery($matchpoint, $record); |
| 708 |
push (@searchstrings, $query) if $query; |
| 709 |
} |
| 595 |
my $op = 'AND'; |
710 |
my $op = 'AND'; |
| 596 |
return join(" $op ",@searchstrings); |
711 |
return join(" $op ", @searchstrings); |
| 597 |
} |
712 |
} |
| 598 |
sub build_simplequery { |
713 |
sub build_simplequery { |
| 599 |
my $element=shift; |
714 |
my ($matchpoint, $record) = @_; |
| 600 |
my $record=shift; |
715 |
|
| 601 |
my @searchstrings; |
716 |
my @searchstrings; |
| 602 |
my ($index,$recorddata)=split /,/,$element; |
717 |
my ($index, $record_data) = split (/,/, $matchpoint); |
| 603 |
if ($recorddata=~/(\d{3})(.*)/) { |
718 |
if ($record_data =~ /(\d{3})(.*)/) { |
| 604 |
my ($tag,$subfields) =($1,$2); |
719 |
my ($tag, $subfields) = ($1, $2); |
| 605 |
foreach my $field ($record->field($tag)){ |
720 |
foreach my $field ($record->field($tag)) { |
| 606 |
if (length($field->as_string("$subfields"))>0){ |
721 |
if (length($field->as_string("$subfields")) > 0) { |
| 607 |
push @searchstrings,"$index:\"".$field->as_string("$subfields")."\""; |
722 |
push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\""); |
| 608 |
} |
723 |
} |
| 609 |
} |
724 |
} |
| 610 |
} |
725 |
} |
| 611 |
my $op = 'AND'; |
726 |
else { |
| 612 |
return join(" $op ",@searchstrings); |
727 |
print "Invalid matchpoint format, invalid marc-field: $matchpoint\n"; |
|
|
728 |
} |
| 729 |
my $op = 'and'; |
| 730 |
return join(" $op ", @searchstrings); |
| 613 |
} |
731 |
} |
| 614 |
sub report_item_errors { |
732 |
sub report_item_errors { |
| 615 |
my $biblionumber = shift; |
733 |
my $biblionumber = shift; |
|
Lines 624-643
sub report_item_errors {
Link Here
|
| 624 |
print $msg, "\n"; |
742 |
print $msg, "\n"; |
| 625 |
} |
743 |
} |
| 626 |
} |
744 |
} |
| 627 |
sub printlog{ |
745 |
sub printlog { |
| 628 |
my $logelements=shift; |
746 |
my $logelements = shift; |
| 629 |
print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n"; |
747 |
print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n"; |
| 630 |
} |
748 |
} |
| 631 |
sub get_heading_fields{ |
749 |
sub get_heading_fields { |
| 632 |
my $headingfields; |
750 |
my $headingfields; |
| 633 |
if ($authtypes){ |
751 |
if ($authtypes) { |
| 634 |
$headingfields = YAML::XS::LoadFile($authtypes); |
752 |
$headingfields = YAML::XS::LoadFile($authtypes); |
| 635 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
753 |
$headingfields = { C4::Context->preference('marcflavour') => $headingfields }; |
| 636 |
$logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields))); |
754 |
$logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields))); |
| 637 |
} |
755 |
} |
| 638 |
unless ($headingfields){ |
756 |
unless ($headingfields) { |
| 639 |
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
757 |
$headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}}); |
| 640 |
$headingfields={C4::Context->preference('marcflavour')=>$headingfields}; |
758 |
$headingfields = { C4::Context->preference('marcflavour') => $headingfields }; |
| 641 |
} |
759 |
} |
| 642 |
return $headingfields; |
760 |
return $headingfields; |
| 643 |
} |
761 |
} |
| 644 |
- |
|
|