View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +28 lines)
Lines 344-349 Unless C<disable_autolink> is passed ModBiblio will relink record headings Link Here
344
to authorities based on settings in the system preferences. This flag allows
344
to authorities based on settings in the system preferences. This flag allows
345
us to not relink records when the authority linker is saving modifications.
345
us to not relink records when the authority linker is saving modifications.
346
346
347
=item C<defer_search_engine_indexing>
348
349
Don't update search index. Useful for bulk updates where this is handled
350
manually for optimization purposes.
351
347
=back
352
=back
348
353
349
Returns 1 on success 0 on failure
354
Returns 1 on success 0 on failure
Lines 353-358 Returns 1 on success 0 on failure Link Here
353
sub ModBiblio {
358
sub ModBiblio {
354
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
359
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
355
    $options //= {};
360
    $options //= {};
361
    my %mod_biblio_marc_options;
362
    $mod_biblio_marc_options{'defer_search_engine_indexing'} =
363
        exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
356
364
357
    if (!$record) {
365
    if (!$record) {
358
        carp 'No record passed to ModBiblio';
366
        carp 'No record passed to ModBiblio';
Lines 416-422 sub ModBiblio { Link Here
416
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
424
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
417
425
418
    # update the MARC record (that now contains biblio and items) with the new record data
426
    # update the MARC record (that now contains biblio and items) with the new record data
419
    ModBiblioMarc( $record, $biblionumber );
427
    ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
420
428
421
    # modify the other koha tables
429
    # modify the other koha tables
422
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
430
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2949-2966 sub _koha_delete_biblio_metadata { Link Here
2949
2957
2950
=head2 ModBiblioMarc
2958
=head2 ModBiblioMarc
2951
2959
2952
  ModBiblioMarc($newrec,$biblionumber);
2960
  ModBiblioMarc($newrec, $biblionumber, $options);
2953
2961
2954
Add MARC XML data for a biblio to koha
2962
Add MARC XML data for a biblio to koha
2955
2963
2956
Function exported, but should NOT be used, unless you really know what you're doing
2964
Function exported, but should NOT be used, unless you really know what you're doing
2957
2965
2966
The C<$options> argument is a hashref with additional parameters:
2967
2968
=over 4
2969
2970
=item C<defer_search_engine_indexing>
2971
2972
Don't update search index. Useful for bulk updates where this is handled
2973
manually for optimization purposes.
2974
2975
=back
2976
2958
=cut
2977
=cut
2959
2978
2960
sub ModBiblioMarc {
2979
sub ModBiblioMarc {
2961
    # pass the MARC::Record to this function, and it will create the records in
2980
    # pass the MARC::Record to this function, and it will create the records in
2962
    # the marcxml field
2981
    # the marcxml field
2963
    my ( $record, $biblionumber ) = @_;
2982
    my ( $record, $biblionumber, $options ) = @_;
2983
    $options //= {};
2984
2964
    if ( !$record ) {
2985
    if ( !$record ) {
2965
        carp 'ModBiblioMarc passed an undefined record';
2986
        carp 'ModBiblioMarc passed an undefined record';
2966
        return;
2987
        return;
Lines 3025-3032 sub ModBiblioMarc { Link Here
3025
    $m_rs->metadata( $record->as_xml_record($encoding) );
3046
    $m_rs->metadata( $record->as_xml_record($encoding) );
3026
    $m_rs->store;
3047
    $m_rs->store;
3027
3048
3028
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3049
    unless (exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'}) {
3029
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3050
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3051
        $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3052
    }
3030
3053
3031
    return $biblionumber;
3054
    return $biblionumber;
3032
}
3055
}
(-)a/misc/migration_tools/bulkmarcimport.pl (-308 / +429 lines)
Lines 23-28 use C4::Biblio qw( Link Here
23
    GetMarcFromKohaField
23
    GetMarcFromKohaField
24
    ModBiblio
24
    ModBiblio
25
    ModBiblioMarc
25
    ModBiblioMarc
26
    GetFrameworkCode
27
    GetMarcBiblio
26
);
28
);
27
use C4::Koha;
29
use C4::Koha;
28
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
30
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
Lines 38-66 use Time::HiRes qw( gettimeofday ); Link Here
38
use Getopt::Long qw( GetOptions );
40
use Getopt::Long qw( GetOptions );
39
use IO::File;
41
use IO::File;
40
use Pod::Usage qw( pod2usage );
42
use Pod::Usage qw( pod2usage );
43
use List::MoreUtils qw(any);
41
44
42
use Koha::Logger;
45
use Koha::Logger;
43
use Koha::Biblios;
46
use Koha::Biblios;
44
use Koha::SearchEngine;
47
use Koha::SearchEngine;
45
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use Koha::Plugins::Handler;
46
50
47
use open qw( :std :encoding(UTF-8) );
51
use open qw( :std :encoding(UTF-8) );
48
binmode( STDOUT, ":encoding(UTF-8)" );
52
binmode(STDOUT, ":encoding(UTF-8)");
49
my ( $input_marc_file, $number, $offset) = ('',0,0);
53
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
50
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
54
my $version;
51
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
55
my $delete;
52
my $cleanisbn = 1;
56
my $test_parameter;
53
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
57
my $skip_marc8_conversion;
58
my $char_encoding;
59
my $verbose;
60
my $commit;
61
my $fk_off;
62
my $format;
63
my $biblios;
64
my $authorities;
65
my $keepids;
66
my $match;
67
my $isbn_check;
68
my $logfile;
69
my $insert;
70
my $filters;
71
my $update;
72
my $all;
73
my $yamlfile;
74
my $authtypes;
75
my $append;
76
my $sourcetag;
77
my $sourcesubfield;
78
my $idmapfl;
79
my $dedup_barcode;
54
my $framework = '';
80
my $framework = '';
55
my $localcust;
81
my $localcust;
56
my $marc_mod_template = '';
82
my $marc_mod_template = '';
57
my $marc_mod_template_id = -1;
83
my $marc_mod_template_id = -1;
58
84
$| = 1;
59
$|=1;
60
85
61
GetOptions(
86
GetOptions(
62
    'commit:f'    => \$commit,
87
    'commit:f' => \$commit,
63
    'file:s'    => \$input_marc_file,
88
    'file:s' => \$input_marc_file,
64
    'n:f' => \$number,
89
    'n:f' => \$number,
65
    'o|offset:f' => \$offset,
90
    'o|offset:f' => \$offset,
66
    'h' => \$version,
91
    'h' => \$version,
Lines 77-98 GetOptions( Link Here
77
    'b|biblios' => \$biblios,
102
    'b|biblios' => \$biblios,
78
    'a|authorities' => \$authorities,
103
    'a|authorities' => \$authorities,
79
    'authtypes:s' => \$authtypes,
104
    'authtypes:s' => \$authtypes,
80
    'filter=s@'     => \$filters,
105
    'filter=s@' => \$filters,
81
    'insert'        => \$insert,
106
    'insert' => \$insert,
82
    'update'        => \$update,
107
    'update' => \$update,
83
    'all'           => \$all,
108
    'all' => \$all,
84
    'match=s@'    => \$match,
109
    'match=s@' => \$match,
85
    'i|isbn' => \$isbn_check,
110
    'i|isbn' => \$isbn_check,
86
    'x:s' => \$sourcetag,
111
    'x:s' => \$sourcetag,
87
    'y:s' => \$sourcesubfield,
112
    'y:s' => \$sourcesubfield,
88
    'idmap:s' => \$idmapfl,
113
    'idmap:s' => \$idmapfl,
89
    'cleanisbn!'     => \$cleanisbn,
114
    'cleanisbn!' => \$cleanisbn,
90
    'yaml:s'        => \$yamlfile,
115
    'yaml:s' => \$yamlfile,
91
    'dedupbarcode' => \$dedup_barcode,
116
    'dedupbarcode' => \$dedup_barcode,
92
    'framework=s' => \$framework,
117
    'framework=s' => \$framework,
93
    'custom:s'    => \$localcust,
118
    'custom:s' => \$localcust,
94
    'marcmodtemplate:s' => \$marc_mod_template,
119
    'marcmodtemplate:s' => \$marc_mod_template,
95
);
120
);
121
96
$biblios ||= !$authorities;
122
$biblios ||= !$authorities;
97
$insert  ||= !$update;
123
$insert  ||= !$update;
98
my $writemode = ($append) ? "a" : "w";
124
my $writemode = ($append) ? "a" : "w";
Lines 104-109 if ($all) { Link Here
104
    $update = 1;
130
    $update = 1;
105
}
131
}
106
132
133
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
134
my $modify_biblio_marc_options = {
135
    defer_search_engine_indexing => $using_elastic_search,
136
    overlay_context => { source => 'bulkmarcimport' }
137
};
138
139
my @search_engine_record_ids;
140
my @search_engine_records;
141
my $indexer;
142
if ($using_elastic_search) {
143
    use Koha::SearchEngine::Elasticsearch::Indexer;
144
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
145
        { index => $authorities ?
146
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
147
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
148
        }
149
    );
150
}
151
107
if ($version || ($input_marc_file eq '')) {
152
if ($version || ($input_marc_file eq '')) {
108
    pod2usage( -verbose => 2 );
153
    pod2usage( -verbose => 2 );
109
    exit;
154
    exit;
Lines 112-124 if( $update && !( $match || $isbn_check ) ) { Link Here
112
    warn "Using -update without -match or -isbn seems to be useless.\n";
157
    warn "Using -update without -match or -isbn seems to be useless.\n";
113
}
158
}
114
159
115
if(defined $localcust) { #local customize module
160
if (defined $localcust) { #local customize module
116
    if(!-e $localcust) {
161
    if (!-e $localcust) {
117
        $localcust= $localcust||'LocalChanges'; #default name
162
        $localcust = $localcust || 'LocalChanges'; #default name
118
        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
163
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
119
        $localcust=~ s/\.pm$//;           #remove extension
164
        $localcust =~ s/\.pm$//;           #remove extension
120
        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
165
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
121
        if(-e $fqcust) {
166
        if (-e $fqcust) {
122
            $localcust= $fqcust;
167
            $localcust= $fqcust;
123
        }
168
        }
124
        else {
169
        else {
Lines 127-133 if(defined $localcust) { #local customize module Link Here
127
        }
172
        }
128
    }
173
    }
129
    require $localcust if $localcust;
174
    require $localcust if $localcust;
130
    $localcust=\&customize if $localcust;
175
    $localcust = \&customize if $localcust;
131
}
176
}
132
177
133
if($marc_mod_template ne '') {
178
if($marc_mod_template ne '') {
Lines 152-219 if($marc_mod_template ne '') { Link Here
152
}
197
}
153
198
154
my $dbh = C4::Context->dbh;
199
my $dbh = C4::Context->dbh;
155
my $heading_fields=get_heading_fields();
200
my $heading_fields = get_heading_fields();
156
157
my $idmapfh;
201
my $idmapfh;
202
158
if (defined $idmapfl) {
203
if (defined $idmapfl) {
159
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
204
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
160
}
205
}
161
206
162
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
207
if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
163
  $sourcetag="910";
208
    $sourcetag = "910";
164
  $sourcesubfield="a";
209
    $sourcesubfield = "a";
165
}
210
}
166
211
167
168
# Disable logging for the biblios and authorities import operation. It would unnecessarily
212
# Disable logging for the biblios and authorities import operation. It would unnecessarily
169
# slow the import
213
# slow the import
170
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
214
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
171
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
215
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
172
216
173
if ($fk_off) {
217
if ($fk_off) {
174
	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
218
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
175
}
219
}
176
220
177
178
if ($delete) {
221
if ($delete) {
179
	if ($biblios){
222
    if ($biblios) {
180
    	print "deleting biblios\n";
223
        print "Deleting biblios\n";
181
        $dbh->do("DELETE FROM biblio");
224
        $dbh->do("DELETE FROM biblio");
182
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
225
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
183
        $dbh->do("DELETE FROM biblioitems");
226
        $dbh->do("DELETE FROM biblioitems");
184
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
227
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
185
        $dbh->do("DELETE FROM items");
228
        $dbh->do("DELETE FROM items");
186
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
229
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
187
	}
230
    }
188
	else {
231
    else {
189
    	print "deleting authorities\n";
232
        print "Deleting authorities\n";
190
    	$dbh->do("truncate auth_header");
233
        $dbh->do("truncate auth_header");
191
	}
234
    }
192
    $dbh->do("truncate zebraqueue");
235
    $dbh->do("truncate zebraqueue");
193
}
236
}
194
237
195
196
197
if ($test_parameter) {
238
if ($test_parameter) {
198
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
239
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
199
}
240
}
200
241
201
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
242
my $batch;
243
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
202
244
203
# The definition of $searcher must be before MARC::Batch->new
245
# The definition of $searcher must be before MARC::Batch->new
204
my $searcher = Koha::SearchEngine::Search->new(
246
my $searcher = Koha::SearchEngine::Search->new(
205
    {
247
    {
206
        index => (
248
        index => (
207
              $authorities
249
            $authorities
208
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
250
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
209
            : $Koha::SearchEngine::BIBLIOS_INDEX
251
            : $Koha::SearchEngine::BIBLIOS_INDEX
210
        )
252
        )
211
    }
253
    }
212
);
254
);
213
255
214
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
256
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
215
my $starttime = gettimeofday;
257
my $starttime = gettimeofday;
216
my $batch;
258
217
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
259
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
218
if (defined $format && $format =~ /XML/i) {
260
if (defined $format && $format =~ /XML/i) {
219
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
261
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
Lines 226-279 if (defined $format && $format =~ /XML/i) { Link Here
226
    #       extract the records, not using regexes to look
268
    #       extract the records, not using regexes to look
227
    #       for <record>.*</record>.
269
    #       for <record>.*</record>.
228
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
270
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
229
    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
271
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
230
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
272
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
231
    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
273
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
232
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
274
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
233
    $batch = MARC::Batch->new( 'XML', $fh );
275
    $batch = MARC::Batch->new('XML', $fh);
234
} else {
276
}
235
    $batch = MARC::Batch->new( 'USMARC', $fh );
277
else {
278
    $batch = MARC::Batch->new('USMARC', $fh);
236
}
279
}
280
237
$batch->warnings_off();
281
$batch->warnings_off();
238
$batch->strict_off();
282
$batch->strict_off();
239
my $i=0;
240
my $commitnum = $commit ? $commit : 50;
283
my $commitnum = $commit ? $commit : 50;
241
my $yamlhash;
284
my $yamlhash;
242
285
243
# Skip file offset
286
# Skip file offset
244
if ( $offset ) {
287
if ($offset) {
245
    print "Skipping file offset: $offset records\n";
288
    print "Skipping file offset: $offset records\n";
246
    $batch->next() while ($offset--);
289
    $batch->next() while ($offset--);
247
}
290
}
248
291
249
my ($tagid,$subfieldid);
292
my ($tagid, $subfieldid);
250
if ($authorities){
293
if ($authorities) {
251
	  $tagid='001';
294
    $tagid = '001';
252
}
295
}
253
else {
296
else {
254
   ( $tagid, $subfieldid ) =
297
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
255
            GetMarcFromKohaField( "biblio.biblionumber" );
298
    $tagid ||= "001";
256
	$tagid||="001";
257
}
299
}
258
300
301
my $sth_isbn;
259
# the SQL query to search on isbn
302
# the SQL query to search on isbn
260
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
303
if ($isbn_check) {
304
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
305
}
261
306
262
my $loghandle;
307
my $loghandle;
263
if ($logfile){
308
if ($logfile) {
264
   $loghandle= IO::File->new($logfile, $writemode) ;
309
    $loghandle= IO::File->new($logfile, $writemode);
265
   print $loghandle "id;operation;status\n";
310
    print $loghandle "id;operation;status\n";
266
}
311
}
267
312
313
my $record_number = 0;
268
my $logger = Koha::Logger->get;
314
my $logger = Koha::Logger->get;
269
my $schema = Koha::Database->schema;
315
my $schema = Koha::Database->schema;
270
$schema->txn_begin;
316
my $marc_records = [];
271
RECORD: while (  ) {
317
RECORD: while (  ) {
272
    my $record;
318
    my $record;
273
    # get records
319
    $record_number++;
320
    # get record
274
    eval { $record = $batch->next() };
321
    eval { $record = $batch->next() };
275
    if ( $@ ) {
322
    if ($@) {
276
        print "Bad MARC record $i: $@ skipped\n";
323
        print "Bad MARC record $record_number: $@ skipped\n";
277
        # FIXME - because MARC::Batch->next() combines grabbing the next
324
        # FIXME - because MARC::Batch->next() combines grabbing the next
278
        # blob and parsing it into one operation, a correctable condition
325
        # blob and parsing it into one operation, a correctable condition
279
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
326
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
Lines 282-560 RECORD: while ( ) { Link Here
282
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
329
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
283
        next;
330
        next;
284
    }
331
    }
285
    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
332
    if ($record) {
286
    last unless ( $record );
333
        # transcode the record to UTF8 if needed & applicable.
287
    $i++;
334
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
288
    if( ($verbose//1)==1 ) { #no dot for verbose==2
335
            my ($guessed_charset, $charset_errors);
289
        print "." . ( $i % 100==0 ? "\n$i" : '' );
336
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
337
            if ($guessed_charset eq 'failed') {
338
                warn "ERROR: failed to perform character conversion for record $record_number\n";
339
                next RECORD;
340
            }
341
        }
342
        SetUTF8Flag($record);
343
        &$localcust($record) if $localcust;
344
        push @{$marc_records}, $record;
345
    }
346
    else {
347
        last;
290
    }
348
    }
349
}
291
350
292
    # transcode the record to UTF8 if needed & applicable.
351
$record_number = 0;
293
    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
352
my $records_total = @{$marc_records};
294
        # FIXME update condition
353
$schema->txn_begin;
295
        my ($guessed_charset, $charset_errors);
354
RECORD: foreach my $record (@{$marc_records}) {
296
         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
355
    $record_number++;
297
        if ($guessed_charset eq 'failed') {
356
    if (($verbose//1) == 1) { #no dot for verbose==2
298
            warn "ERROR: failed to perform character conversion for record $i\n";
357
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
299
            next RECORD;            
300
        }
301
    }
358
    }
302
    SetUTF8Flag($record);
359
303
    if($marc_mod_template_id > 0) {
360
    if ($marc_mod_template_id > 0) {
304
    print "Modifying MARC\n" if $verbose;
361
        print "Modifying MARC\n" if $verbose;
305
    ModifyRecordWithTemplate( $marc_mod_template_id, $record );
362
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
306
    }
363
    }
307
    &$localcust($record) if $localcust;
364
308
    my $isbn;
365
    my $isbn;
309
    # remove trailing - in isbn (only for biblios, of course)
366
    # remove trailing - in isbn (only for biblios, of course)
310
    if( $biblios ) {
367
    if ($biblios && ($cleanisbn || $isbn_check)) {
311
        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
368
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
312
        my $field = $record->field($tag);
369
        my $field = $record->field($tag);
313
        $isbn = $field && $field->subfield('a');
370
        $isbn = $field && $field->subfield('a');
314
        if ( $isbn && $cleanisbn ) {
371
        if ($isbn && $cleanisbn) {
315
            $isbn =~ s/-//g;
372
            $isbn =~ s/-//g;
316
            $field->update('a' => $isbn);
373
            $field->update('a' => $isbn);
317
        }
374
        }
318
    }
375
    }
319
    my $id;
320
    # search for duplicates (based on Local-number)
376
    # search for duplicates (based on Local-number)
321
    my $originalid;
377
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
322
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
378
    my $matched_record_id = undef;
323
    if ($match) {
379
    if ($match) {
324
        require C4::Search;
380
        require C4::Search;
325
        my $query = build_query( $match, $record );
381
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
326
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
382
        my $query = build_query($match, $record);
327
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
383
        $logger->debug("Bulkmarcimport: $query");
384
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
328
        # changed to warn so able to continue with one broken record
385
        # changed to warn so able to continue with one broken record
329
        if ( defined $error ) {
386
        if (defined $error) {
330
            warn "unable to search the database for duplicates : $error";
387
            warn "unable to search the database for duplicates : $error";
331
            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
388
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
332
            next RECORD;
389
            next RECORD;
333
        }
390
        }
334
        if ( $results && scalar(@$results) == 1 ) {
391
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
335
            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
392
        # sub SimpleSearch could return undefined, but only on error, so
336
            SetUTF8Flag($marcrecord);
393
        # should not really need to safeguard here, but do so anyway
337
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
394
        $results //= [];
338
            if ( $authorities && $marcFlavour ) {
395
        if (@{$results} == 1) {
339
                #Skip if authority in database is the same as the on in database
396
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
340
                if ( $marcrecord->field('005') && $record->field('005') &&
397
            SetUTF8Flag($matched_record);
341
                     $marcrecord->field('005')->data && $record->field('005')->data &&
398
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
342
                     $marcrecord->field('005')->data >= $record->field('005')->data ) {
399
400
            if ($authorities && $marc_flavour) {
401
                #Skip if authority in database is the same or newer than the incoming record
402
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
343
                    if ($yamlfile) {
403
                    if ($yamlfile) {
344
                        $yamlhash->{$originalid}->{'authid'} = $id;
404
                        $yamlhash->{$originalid} = YAMLFileEntry(
345
405
                            $matched_record,
346
                        # we recover all subfields of the heading authorities
406
                            $matched_record_id,
347
                        my @subfields;
407
                            0
348
                        foreach my $field ( $marcrecord->field("2..") ) {
408
                        );
349
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
350
                        }
351
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
352
                        $yamlhash->{$originalid}->{'updated'} = 0;
353
                    }
409
                    }
354
                    next;
410
                    next;
355
                }
411
                }
356
            }
412
            }
357
        } elsif ( $results && scalar(@$results) > 1 ) {
358
            $logger->debug("more than one match for $query");
359
        } else {
360
            $logger->debug("nomatch for $query");
361
        }
413
        }
362
    }
414
        elsif(@{$results} > 1) {
363
    if ($keepids && $originalid) {
415
            $logger->debug("More than one match for: $query");
416
        }
417
        else {
418
            $logger->debug("No match for: $query");
419
        }
420
421
        if ($keepids && $originalid) {
364
            my $storeidfield;
422
            my $storeidfield;
365
            if ( length($keepids) == 3 ) {
423
            if (length($keepids) == 3) {
366
                $storeidfield = MARC::Field->new( $keepids, $originalid );
424
                $storeidfield = MARC::Field->new($keepids, $originalid);
367
            } else {
425
            } else {
368
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
426
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
369
            }
427
            }
370
            $record->insert_fields_ordered($storeidfield);
428
            $record->insert_fields_ordered($storeidfield);
371
            $record->delete_field( $record->field($tagid) );
429
            $record->delete_field($record->field($tagid));
430
        }
372
    }
431
    }
432
373
    foreach my $stringfilter (@$filters) {
433
    foreach my $stringfilter (@$filters) {
374
        if ( length($stringfilter) == 3 ) {
434
        if (length($stringfilter) == 3) {
375
            foreach my $field ( $record->field($stringfilter) ) {
435
            foreach my $field ($record->field($stringfilter)) {
376
                $record->delete_field($field);
436
                $record->delete_field($field);
377
                $logger->debug("removed : ", $field->as_string);
437
                $logger->debug("Removed: ", $field->as_string);
378
            }
438
            }
379
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
439
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
380
            my $removetag = $1;
440
            my $removetag = $1;
381
            my $removesubfield = $2;
441
            my $removesubfield = $2;
382
            my $removematch = $3;
442
            my $removematch = $3;
383
            if ( ( $removetag > "010" ) && $removesubfield ) {
443
            if (($removetag > "010") && $removesubfield) {
384
                foreach my $field ( $record->field($removetag) ) {
444
                foreach my $field ($record->field($removetag)) {
385
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
445
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
386
                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));
446
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
387
                }
447
                }
388
            }
448
            }
389
        }
449
        }
390
    }
450
    }
391
    unless ($test_parameter) {
451
    unless ($test_parameter) {
392
        if ($authorities){
452
        if ($authorities) {
393
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
453
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
394
            my $authid= ($id?$id:GuessAuthId($record));
454
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
395
            if ($authid && GetAuthority($authid) && $update ){
455
            if ($authid && GetAuthority($authid) && $update) {
396
            ## Authority has an id and is in database : Replace
456
            ## Authority has an id and is in database : Replace
397
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
457
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
398
                if ($@){
458
                if ($@) {
399
                    warn "Problem with authority $authid Cannot Modify";
459
                    warn "Problem with authority $authid Cannot Modify";
400
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
460
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
401
                }
461
                }
402
				else{
462
                else{
403
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
463
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
404
				}
464
                }
405
            }  
465
            }
406
	        else {
466
            else {
407
            ## True insert in database
467
            ## True insert in database
408
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
468
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
409
                if ($@){
469
                if ($@) {
410
                    warn "Problem with authority $authid Cannot Add".$@;
470
                    warn "Problem with authority $originalid Cannot Add ".$@;
411
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
471
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
412
                }
472
                }
413
   				else{
473
                else {
414
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
474
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
415
				}
475
                }
416
 	        }
476
417
            if ($yamlfile) {
418
            $yamlhash->{$originalid}->{'authid'} = $authid;
419
            my @subfields;
420
            foreach my $field ( $record->field("2..") ) {
421
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
422
            }
477
            }
423
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
478
            if ($yamlfile) {
424
            $yamlhash->{$originalid}->{'updated'} = 1;
479
                $yamlhash->{$originalid} = YAMLFileEntry(
480
                    $record,
481
                    $authid,
482
                    1 #@FIXME: Really always updated?
483
                );
425
            }
484
            }
426
        }
485
        }
427
        else {
486
        else {
428
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
487
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
429
            $biblionumber = $id;
430
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
488
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
431
            if (!$biblionumber && $isbn_check && $isbn) {
489
            if (!$matched_record_id && $isbn_check && $isbn) {
432
    #         warn "search ISBN : $isbn";
433
                $sth_isbn->execute($isbn);
490
                $sth_isbn->execute($isbn);
434
                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
491
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
492
            }
493
494
            if (defined $idmapfl && $matched_record_id) {
495
                if ($sourcetag < "010") {
496
                    if ($record->field($sourcetag)) {
497
                        my $source = $record->field($sourcetag)->data();
498
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
499
                    }
500
                }
501
                else {
502
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
503
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
504
                }
435
            }
505
            }
436
        	if (defined $idmapfl) {
506
437
			 	if ($sourcetag < "010"){
507
            # create biblio, unless we already have it ( either match or isbn )
438
					if ($record->field($sourcetag)){
508
            if ($matched_record_id) {
439
					  my $source = $record->field($sourcetag)->data();
509
                # TODO: Implement also for authority records!
440
                      printf($idmapfh "%s|%s\n",$source,$biblionumber);
441
					}
442
			    } else {
443
					my $source=$record->subfield($sourcetag,$sourcesubfield);
444
                    printf($idmapfh "%s|%s\n",$source,$biblionumber);
445
			  }
446
			}
447
					# create biblio, unless we already have it ( either match or isbn )
448
            if ($biblionumber) {
449
                eval{
510
                eval{
450
                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
511
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
451
                };
512
                };
452
                if ($update) {
513
                if ($update) {
453
                    eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) };
514
                    my $success;
515
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
454
                    if ($@) {
516
                    if ($@) {
455
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
517
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
456
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
518
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
457
                        next RECORD;
519
                        next RECORD;
458
                    } else {
459
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
460
                    }
520
                    }
461
                } else {
521
                    elsif (!$success) {
462
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
522
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
463
                }
523
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
464
            } else {
465
                if ($insert) {
466
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
467
                    if ($@) {
468
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
469
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
470
                        next RECORD;
524
                        next RECORD;
471
                    } else {
472
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
473
                    }
525
                    }
474
                } else {
526
                    else {
475
                    warn "WARNING: Updating record ".($id||$originalid)." failed";
527
                        $record_id = $matched_record_id;
476
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
528
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
529
                    }
530
                }
531
                else {
532
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
533
                }
534
            }
535
            elsif ($insert) {
536
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
537
                if ($@) {
538
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
539
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
477
                    next RECORD;
540
                    next RECORD;
478
                }
541
                }
542
                else {
543
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
544
                }
479
            }
545
            }
480
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
546
            else {
481
            my $error_adding = $@;
547
                warn "WARNING: Updating record ".($originalid)." failed";
482
            # Work on a clone so that if there are real errors, we can maybe
548
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
483
            # fix them up later.
484
			my $clone_record = $record->clone();
485
            C4::Biblio::_strip_item_fields($clone_record, '');
486
            # This sets the marc fields if there was an error, and also calls
487
            # defer_marc_save.
488
            ModBiblioMarc( $clone_record, $biblionumber );
489
            if ( $error_adding ) {
490
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
491
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
492
                # if we failed because of an exception, assume that 
493
                # the MARC columns in biblioitems were not set.
494
                next RECORD;
549
                next RECORD;
495
            }
550
            }
496
 			else{
551
            my $record_has_added_items = 0;
497
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
552
            if ($record_id) {
498
			}
553
                $yamlhash->{$originalid} = $record_id if $yamlfile;
499
            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
554
                # TODO: Add option for skipping items?
500
                # Find the record called 'barcode'
555
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
501
                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
556
                $record_has_added_items = @{$itemnumbers_ref};
502
                # Now remove any items that didn't have a duplicate_barcode error,
557
                my $error_adding = $@;
503
                # erase the barcodes on items that did, and re-add those items.
558
                # Work on a clone so that if there are real errors, we can maybe
504
                my %dupes;
559
                # fix them up later.
505
                foreach my $i (0 .. $#{$errors_ref}) {
560
                my $clone_record = $record->clone();
506
                    my $ref = $errors_ref->[$i];
561
                C4::Biblio::_strip_item_fields($clone_record, $framework);
507
                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
562
                # This sets the marc fields if there was an error, and also calls
508
                        $dupes{$ref->{item_sequence}} = 1;
563
                # defer_marc_save.
509
                        # Delete the error message because we're going to
564
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
510
                        # retry this one.
565
                if ($error_adding) {
511
                        delete $errors_ref->[$i];
566
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
512
                    }
567
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
513
                }
514
                my $seq = 0;
515
                foreach my $field ($record->field($tag)) {
516
                    $seq++;
517
                    if ($dupes{$seq}) {
518
                        # Here we remove the barcode
519
                        $field->delete_subfield(code => $sub);
520
                    } else {
521
                        # otherwise we delete the field because we don't want
522
                        # two of them
523
                        $record->delete_fields($field);
524
                    }
525
                }
526
                # Now re-add the record as before, adding errors to the prev list
527
                my $more_errors;
528
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
529
                if ( $@ ) {
530
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
531
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
532
                    # if we failed because of an exception, assume that
568
                    # if we failed because of an exception, assume that
533
                    # the MARC columns in biblioitems were not set.
569
                    # the MARC columns in biblioitems were not set.
534
                    ModBiblioMarc( $record, $biblionumber );
535
                    next RECORD;
570
                    next RECORD;
536
                } else {
537
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
538
                }
571
                }
539
                push @$errors_ref, @{ $more_errors };
572
                else {
540
            }
573
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
541
            if ($#{ $errors_ref } > -1) {
574
                }
542
                report_item_errors($biblionumber, $errors_ref);
575
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
576
                    # Find the record called 'barcode'
577
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
578
                    # Now remove any items that didn't have a duplicate_barcode error,
579
                    # erase the barcodes on items that did, and re-add those items.
580
                    my %dupes;
581
                    # FIXME: This could cause array out of bounds because shifting down rest of items on array item delete?
582
                    foreach my $i (0 .. $#{$errors_ref}) {
583
                        my $ref = $errors_ref->[$i];
584
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
585
                            $dupes{$ref->{item_sequence}} = 1;
586
                            # Delete the error message because we're going to
587
                            # retry this one.
588
                            delete $errors_ref->[$i];
589
                        }
590
                    }
591
                    my $seq = 0;
592
                    foreach my $field ($record->field($tag)) {
593
                        $seq++;
594
                        if ($dupes{$seq}) {
595
                            # Here we remove the barcode
596
                            $field->delete_subfield(code => $sub);
597
                        }
598
                        else {
599
                            # otherwise we delete the field because we don't want
600
                            # two of them
601
                            $record->delete_fields($field);
602
                        }
603
                    }
604
                    # Now re-add the record as before, adding errors to the prev list
605
                    my $more_errors;
606
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
607
                    $record_has_added_items ||= @{$itemnumbers_ref};
608
                    if ($@) {
609
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
610
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
611
                        # if we failed because of an exception, assume that
612
                        # the MARC columns in biblioitems were not set.
613
614
                        # @FIXME: Why do we save here without stripping items? Besides,
615
                        # save with stripped items has already been performed
616
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
617
                        next RECORD;
618
                    }
619
                    else {
620
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
621
                    }
622
                    push @$errors_ref, @{$more_errors};
623
                }
624
                if (@{$errors_ref}) {
625
                    report_item_errors($record_id, $errors_ref);
626
                }
627
                C4::Biblio::_strip_item_fields($record, $framework);
628
                if ($record_has_added_items || $matched_record_id) {
629
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
630
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
631
                }
632
                push @search_engine_record_ids, $record_id;
633
                push @search_engine_records, $record;
543
            }
634
            }
544
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
545
        }
635
        }
546
        if ( 0 == $i % $commitnum ) {
636
        if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) {
547
            $schema->txn_commit;
637
            $schema->txn_commit;
548
            $schema->txn_begin;
638
            $schema->txn_begin;
639
            if ($indexer) {
640
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
641
                @search_engine_record_ids = ();
642
                @search_engine_records = ();
643
            }
549
        }
644
        }
550
    }
645
    }
551
    print $record->as_formatted()."\n" if ($verbose//0)==2;
646
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
552
    last if $i == $number;
647
    last if $record_number == $number;
553
}
648
}
554
$schema->txn_commit;
649
$schema->txn_commit;
555
650
556
if ($fk_off) {
651
if ($fk_off) {
557
	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
652
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
558
}
653
}
559
654
560
# Restore CataloguingLog and AuthoritiesLog
655
# Restore CataloguingLog and AuthoritiesLog
Lines 562-572 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; Link Here
562
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
657
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
563
658
564
my $timeneeded = gettimeofday - $starttime;
659
my $timeneeded = gettimeofday - $starttime;
565
print "\n$i MARC records done in $timeneeded seconds\n";
660
print "\n$record_number MARC records done in $timeneeded seconds\n";
566
if ($logfile){
661
if ($logfile) {
567
  print $loghandle "file : $input_marc_file\n";
662
    print $loghandle "file : $input_marc_file\n";
568
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
663
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
569
  $loghandle->close;
664
    $loghandle->close;
570
}
665
}
571
if ($yamlfile) {
666
if ($yamlfile) {
572
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
667
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
Lines 574-620 if ($yamlfile) { Link Here
574
}
669
}
575
exit 0;
670
exit 0;
576
671
577
sub GetRecordId{
672
sub YAMLFileEntry {
578
	my $marcrecord=shift;
673
    my ($record, $record_id, $updated) = @_;
579
	my $tag=shift;
674
580
	my $subfield=shift;
675
    my $entry = {
581
	my $id;
676
        authid => $record_id
582
	if ($tag lt "010"){
677
    };
583
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
678
584
	} 
679
    # we recover all subfields of the heading authorities
585
	elsif ($subfield){
680
    my @subfields;
586
		if ($marcrecord->field($tag)){
681
    foreach my $field ($record->field("2..")) {
587
			return $marcrecord->subfield($tag,$subfield);
682
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
588
		}
683
    }
589
	}
684
    $entry->{'subfields'} = \@subfields;
590
	return $id;
685
    $entry->{'updated'} = $updated;
686
687
    return $entry;
591
}
688
}
592
sub build_query {
689
593
	my $match = shift;
690
sub RecordRevisionIsGtOrEq {
594
	my $record=shift;
691
    my ($record_a, $record_b) = @_;
595
        my @searchstrings;
692
    return $record_a->field('005') && $record_b->field('005') &&
596
	foreach my $matchingpoint (@$match){
693
    $record_a->field('005')->data && $record_b->field('005')->data &&
597
	  my $string = build_simplequery($matchingpoint,$record);
694
    $record_a->field('005')->data >= $record_b->field('005')->data;
598
	  push @searchstrings,$string if (length($string)>0);
695
}
696
697
sub GetRecordId {
698
    my $marcrecord = shift;
699
    my $tag = shift;
700
    my $subfield = shift;
701
    if ($tag lt "010") {
702
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
703
    }
704
    elsif ($subfield) {
705
        if ($marcrecord->field($tag)) {
706
            return $marcrecord->subfield($tag, $subfield);
599
        }
707
        }
708
    }
709
    return undef;
710
}
711
sub build_query {
712
    my ($match, $record) = @_;
713
    my @searchstrings;
714
715
    foreach my $matchpoint (@$match) {
716
        my $query = build_simplequery($matchpoint, $record);
717
        push (@searchstrings, $query) if $query;
718
    }
600
    my $op = 'and';
719
    my $op = 'and';
601
    return join(" $op ",@searchstrings);
720
    return join(" $op ", @searchstrings);
602
}
721
}
603
sub build_simplequery {
722
sub build_simplequery {
604
	my $element=shift;
723
    my ($matchpoint, $record) = @_;
605
	my $record=shift;
724
606
    my @searchstrings;
725
    my @searchstrings;
607
    my ($index,$recorddata)=split /,/,$element;
726
    my ($index, $record_data) = split (/,/, $matchpoint);
608
    if ($recorddata=~/(\d{3})(.*)/) {
727
    if ($record_data =~ /(\d{3})(.*)/) {
609
        my ($tag,$subfields) =($1,$2);
728
        my ($tag, $subfields) = ($1, $2);
610
        foreach my $field ($record->field($tag)){
729
        foreach my $field ($record->field($tag)) {
611
		  if (length($field->as_string("$subfields"))>0){
730
            if (length($field->as_string("$subfields")) > 0) {
612
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
731
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
613
		  }
732
            }
614
        }
733
        }
615
    }
734
    }
735
    else {
736
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
737
    }
616
    my $op = 'and';
738
    my $op = 'and';
617
    return join(" $op ",@searchstrings);
739
    return join(" $op ", @searchstrings);
618
}
740
}
619
sub report_item_errors {
741
sub report_item_errors {
620
    my $biblionumber = shift;
742
    my $biblionumber = shift;
Lines 629-648 sub report_item_errors { Link Here
629
        print $msg, "\n";
751
        print $msg, "\n";
630
    }
752
    }
631
}
753
}
632
sub printlog{
754
sub printlog {
633
	my $logelements=shift;
755
    my $logelements = shift;
634
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
756
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
635
}
757
}
636
sub get_heading_fields{
758
sub get_heading_fields {
637
    my $headingfields;
759
    my $headingfields;
638
    if ($authtypes){
760
    if ($authtypes) {
639
        $headingfields = YAML::XS::LoadFile($authtypes);
761
        $headingfields = YAML::XS::LoadFile($authtypes);
640
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
762
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
641
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
763
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
642
    }
764
    }
643
    unless ($headingfields){
765
    unless ($headingfields) {
644
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
766
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
645
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
767
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
646
    }
768
    }
647
    return $headingfields;
769
    return $headingfields;
648
}
770
}
649
- 

Return to bug 29440