View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +28 lines)
Lines 344-349 Unless C<disable_autolink> is passed ModBiblio will relink record headings Link Here
344
to authorities based on settings in the system preferences. This flag allows
344
to authorities based on settings in the system preferences. This flag allows
345
us to not relink records when the authority linker is saving modifications.
345
us to not relink records when the authority linker is saving modifications.
346
346
347
=item C<defer_search_engine_indexing>
348
349
Don't update search index. Useful for bulk updates where this is handled
350
manually for optimization purposes.
351
347
=back
352
=back
348
353
349
Returns 1 on success 0 on failure
354
Returns 1 on success 0 on failure
Lines 353-358 Returns 1 on success 0 on failure Link Here
353
sub ModBiblio {
358
sub ModBiblio {
354
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
359
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
355
    $options //= {};
360
    $options //= {};
361
    my %mod_biblio_marc_options;
362
    $mod_biblio_marc_options{'defer_search_engine_indexing'} =
363
        exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
356
364
357
    if (!$record) {
365
    if (!$record) {
358
        carp 'No record passed to ModBiblio';
366
        carp 'No record passed to ModBiblio';
Lines 416-422 sub ModBiblio { Link Here
416
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
424
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
417
425
418
    # update the MARC record (that now contains biblio and items) with the new record data
426
    # update the MARC record (that now contains biblio and items) with the new record data
419
    ModBiblioMarc( $record, $biblionumber );
427
    ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
420
428
421
    # modify the other koha tables
429
    # modify the other koha tables
422
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
430
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2949-2966 sub _koha_delete_biblio_metadata { Link Here
2949
2957
2950
=head2 ModBiblioMarc
2958
=head2 ModBiblioMarc
2951
2959
2952
  ModBiblioMarc($newrec,$biblionumber);
2960
  ModBiblioMarc($newrec, $biblionumber, $options);
2953
2961
2954
Add MARC XML data for a biblio to koha
2962
Add MARC XML data for a biblio to koha
2955
2963
2956
Function exported, but should NOT be used, unless you really know what you're doing
2964
Function exported, but should NOT be used, unless you really know what you're doing
2957
2965
2966
The C<$options> argument is a hashref with additional parameters:
2967
2968
=over 4
2969
2970
=item C<defer_search_engine_indexing>
2971
2972
Don't update search index. Useful for bulk updates where this is handled
2973
manually for optimization purposes.
2974
2975
=back
2976
2958
=cut
2977
=cut
2959
2978
2960
sub ModBiblioMarc {
2979
sub ModBiblioMarc {
2961
    # pass the MARC::Record to this function, and it will create the records in
2980
    # pass the MARC::Record to this function, and it will create the records in
2962
    # the marcxml field
2981
    # the marcxml field
2963
    my ( $record, $biblionumber ) = @_;
2982
    my ( $record, $biblionumber, $options ) = @_;
2983
    $options //= {};
2984
2964
    if ( !$record ) {
2985
    if ( !$record ) {
2965
        carp 'ModBiblioMarc passed an undefined record';
2986
        carp 'ModBiblioMarc passed an undefined record';
2966
        return;
2987
        return;
Lines 3025-3032 sub ModBiblioMarc { Link Here
3025
    $m_rs->metadata( $record->as_xml_record($encoding) );
3046
    $m_rs->metadata( $record->as_xml_record($encoding) );
3026
    $m_rs->store;
3047
    $m_rs->store;
3027
3048
3028
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3049
    unless (exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'}) {
3029
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3050
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3051
        $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3052
    }
3030
3053
3031
    return $biblionumber;
3054
    return $biblionumber;
3032
}
3055
}
(-)a/misc/migration_tools/bulkmarcimport.pl (-307 / +429 lines)
Lines 23-28 use C4::Biblio qw( Link Here
23
    GetMarcFromKohaField
23
    GetMarcFromKohaField
24
    ModBiblio
24
    ModBiblio
25
    ModBiblioMarc
25
    ModBiblioMarc
26
    GetFrameworkCode
27
    GetMarcBiblio
26
);
28
);
27
use C4::Koha;
29
use C4::Koha;
28
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
30
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
Lines 38-66 use Time::HiRes qw( gettimeofday ); Link Here
38
use Getopt::Long qw( GetOptions );
40
use Getopt::Long qw( GetOptions );
39
use IO::File;
41
use IO::File;
40
use Pod::Usage qw( pod2usage );
42
use Pod::Usage qw( pod2usage );
43
use List::MoreUtils qw(any);
41
44
42
use Koha::Logger;
45
use Koha::Logger;
43
use Koha::Biblios;
46
use Koha::Biblios;
44
use Koha::SearchEngine;
47
use Koha::SearchEngine;
45
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use Koha::Plugins::Handler;
46
50
47
use open qw( :std :encoding(UTF-8) );
51
use open qw( :std :encoding(UTF-8) );
48
binmode( STDOUT, ":encoding(UTF-8)" );
52
binmode(STDOUT, ":encoding(UTF-8)");
49
my ( $input_marc_file, $number, $offset) = ('',0,0);
53
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
50
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
54
my $version;
51
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
55
my $delete;
52
my $cleanisbn = 1;
56
my $test_parameter;
53
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
57
my $skip_marc8_conversion;
58
my $char_encoding;
59
my $verbose;
60
my $commit;
61
my $fk_off;
62
my $format;
63
my $biblios;
64
my $authorities;
65
my $keepids;
66
my $match;
67
my $isbn_check;
68
my $logfile;
69
my $insert;
70
my $filters;
71
my $update;
72
my $all;
73
my $yamlfile;
74
my $authtypes;
75
my $append;
76
my $sourcetag;
77
my $sourcesubfield;
78
my $idmapfl;
79
my $dedup_barcode;
54
my $framework = '';
80
my $framework = '';
55
my $localcust;
81
my $localcust;
56
my $marc_mod_template = '';
82
my $marc_mod_template = '';
57
my $marc_mod_template_id = -1;
83
my $marc_mod_template_id = -1;
58
84
$| = 1;
59
$|=1;
60
85
61
GetOptions(
86
GetOptions(
62
    'commit:f'    => \$commit,
87
    'commit:f' => \$commit,
63
    'file:s'    => \$input_marc_file,
88
    'file:s' => \$input_marc_file,
64
    'n:f' => \$number,
89
    'n:f' => \$number,
65
    'o|offset:f' => \$offset,
90
    'o|offset:f' => \$offset,
66
    'h' => \$version,
91
    'h' => \$version,
Lines 77-98 GetOptions( Link Here
77
    'b|biblios' => \$biblios,
102
    'b|biblios' => \$biblios,
78
    'a|authorities' => \$authorities,
103
    'a|authorities' => \$authorities,
79
    'authtypes:s' => \$authtypes,
104
    'authtypes:s' => \$authtypes,
80
    'filter=s@'     => \$filters,
105
    'filter=s@' => \$filters,
81
    'insert'        => \$insert,
106
    'insert' => \$insert,
82
    'update'        => \$update,
107
    'update' => \$update,
83
    'all'           => \$all,
108
    'all' => \$all,
84
    'match=s@'    => \$match,
109
    'match=s@' => \$match,
85
    'i|isbn' => \$isbn_check,
110
    'i|isbn' => \$isbn_check,
86
    'x:s' => \$sourcetag,
111
    'x:s' => \$sourcetag,
87
    'y:s' => \$sourcesubfield,
112
    'y:s' => \$sourcesubfield,
88
    'idmap:s' => \$idmapfl,
113
    'idmap:s' => \$idmapfl,
89
    'cleanisbn!'     => \$cleanisbn,
114
    'cleanisbn!' => \$cleanisbn,
90
    'yaml:s'        => \$yamlfile,
115
    'yaml:s' => \$yamlfile,
91
    'dedupbarcode' => \$dedup_barcode,
116
    'dedupbarcode' => \$dedup_barcode,
92
    'framework=s' => \$framework,
117
    'framework=s' => \$framework,
93
    'custom:s'    => \$localcust,
118
    'custom:s' => \$localcust,
94
    'marcmodtemplate:s' => \$marc_mod_template,
119
    'marcmodtemplate:s' => \$marc_mod_template,
95
);
120
);
121
96
$biblios ||= !$authorities;
122
$biblios ||= !$authorities;
97
$insert  ||= !$update;
123
$insert  ||= !$update;
98
my $writemode = ($append) ? "a" : "w";
124
my $writemode = ($append) ? "a" : "w";
Lines 104-109 if ($all) { Link Here
104
    $update = 1;
130
    $update = 1;
105
}
131
}
106
132
133
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
134
my $modify_biblio_marc_options = {
135
    defer_search_engine_indexing => $using_elastic_search,
136
    overlay_context => { source => 'bulkmarcimport' }
137
};
138
139
my @search_engine_record_ids;
140
my @search_engine_records;
141
my $indexer;
142
if ($using_elastic_search) {
143
    use Koha::SearchEngine::Elasticsearch::Indexer;
144
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
145
        { index => $authorities ?
146
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
147
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
148
        }
149
    );
150
}
151
107
if ($version || ($input_marc_file eq '')) {
152
if ($version || ($input_marc_file eq '')) {
108
    pod2usage( -verbose => 2 );
153
    pod2usage( -verbose => 2 );
109
    exit;
154
    exit;
Lines 112-124 if( $update && !( $match || $isbn_check ) ) { Link Here
112
    warn "Using -update without -match or -isbn seems to be useless.\n";
157
    warn "Using -update without -match or -isbn seems to be useless.\n";
113
}
158
}
114
159
115
if(defined $localcust) { #local customize module
160
if (defined $localcust) { #local customize module
116
    if(!-e $localcust) {
161
    if (!-e $localcust) {
117
        $localcust= $localcust||'LocalChanges'; #default name
162
        $localcust = $localcust || 'LocalChanges'; #default name
118
        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
163
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
119
        $localcust=~ s/\.pm$//;           #remove extension
164
        $localcust =~ s/\.pm$//;           #remove extension
120
        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
165
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
121
        if(-e $fqcust) {
166
        if (-e $fqcust) {
122
            $localcust= $fqcust;
167
            $localcust= $fqcust;
123
        }
168
        }
124
        else {
169
        else {
Lines 127-133 if(defined $localcust) { #local customize module Link Here
127
        }
172
        }
128
    }
173
    }
129
    require $localcust if $localcust;
174
    require $localcust if $localcust;
130
    $localcust=\&customize if $localcust;
175
    $localcust = \&customize if $localcust;
131
}
176
}
132
177
133
if($marc_mod_template ne '') {
178
if($marc_mod_template ne '') {
Lines 152-219 if($marc_mod_template ne '') { Link Here
152
}
197
}
153
198
154
my $dbh = C4::Context->dbh;
199
my $dbh = C4::Context->dbh;
155
my $heading_fields=get_heading_fields();
200
my $heading_fields = get_heading_fields();
156
157
my $idmapfh;
201
my $idmapfh;
202
158
if (defined $idmapfl) {
203
if (defined $idmapfl) {
159
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
204
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
160
}
205
}
161
206
162
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
207
if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
163
  $sourcetag="910";
208
    $sourcetag = "910";
164
  $sourcesubfield="a";
209
    $sourcesubfield = "a";
165
}
210
}
166
211
167
168
# Disable logging for the biblios and authorities import operation. It would unnecessarily
212
# Disable logging for the biblios and authorities import operation. It would unnecessarily
169
# slow the import
213
# slow the import
170
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
214
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
171
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
215
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
172
216
173
if ($fk_off) {
217
if ($fk_off) {
174
	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
218
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
175
}
219
}
176
220
177
178
if ($delete) {
221
if ($delete) {
179
	if ($biblios){
222
    if ($biblios) {
180
    	print "deleting biblios\n";
223
        print "Deleting biblios\n";
181
        $dbh->do("DELETE FROM biblio");
224
        $dbh->do("DELETE FROM biblio");
182
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
225
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
183
        $dbh->do("DELETE FROM biblioitems");
226
        $dbh->do("DELETE FROM biblioitems");
184
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
227
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
185
        $dbh->do("DELETE FROM items");
228
        $dbh->do("DELETE FROM items");
186
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
229
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
187
	}
230
    }
188
	else {
231
    else {
189
    	print "deleting authorities\n";
232
        print "Deleting authorities\n";
190
    	$dbh->do("truncate auth_header");
233
        $dbh->do("truncate auth_header");
191
	}
234
    }
192
    $dbh->do("truncate zebraqueue");
235
    $dbh->do("truncate zebraqueue");
193
}
236
}
194
237
195
196
197
if ($test_parameter) {
238
if ($test_parameter) {
198
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
239
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
199
}
240
}
200
241
201
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
242
my $batch;
243
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
202
244
203
# The definition of $searcher must be before MARC::Batch->new
245
# The definition of $searcher must be before MARC::Batch->new
204
my $searcher = Koha::SearchEngine::Search->new(
246
my $searcher = Koha::SearchEngine::Search->new(
205
    {
247
    {
206
        index => (
248
        index => (
207
              $authorities
249
            $authorities
208
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
250
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
209
            : $Koha::SearchEngine::BIBLIOS_INDEX
251
            : $Koha::SearchEngine::BIBLIOS_INDEX
210
        )
252
        )
211
    }
253
    }
212
);
254
);
213
255
214
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
256
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
215
my $starttime = gettimeofday;
257
my $starttime = gettimeofday;
216
my $batch;
258
217
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
259
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
218
if (defined $format && $format =~ /XML/i) {
260
if (defined $format && $format =~ /XML/i) {
219
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
261
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
Lines 226-279 if (defined $format && $format =~ /XML/i) { Link Here
226
    #       extract the records, not using regexes to look
268
    #       extract the records, not using regexes to look
227
    #       for <record>.*</record>.
269
    #       for <record>.*</record>.
228
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
270
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
229
    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
271
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
230
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
272
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
231
    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
273
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
232
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
274
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
233
    $batch = MARC::Batch->new( 'XML', $fh );
275
    $batch = MARC::Batch->new('XML', $fh);
234
} else {
276
}
235
    $batch = MARC::Batch->new( 'USMARC', $fh );
277
else {
278
    $batch = MARC::Batch->new('USMARC', $fh);
236
}
279
}
280
237
$batch->warnings_off();
281
$batch->warnings_off();
238
$batch->strict_off();
282
$batch->strict_off();
239
my $i=0;
240
my $commitnum = $commit ? $commit : 50;
283
my $commitnum = $commit ? $commit : 50;
241
my $yamlhash;
284
my $yamlhash;
242
285
243
# Skip file offset
286
# Skip file offset
244
if ( $offset ) {
287
if ($offset) {
245
    print "Skipping file offset: $offset records\n";
288
    print "Skipping file offset: $offset records\n";
246
    $batch->next() while ($offset--);
289
    $batch->next() while ($offset--);
247
}
290
}
248
291
249
my ($tagid,$subfieldid);
292
my ($tagid, $subfieldid);
250
if ($authorities){
293
if ($authorities) {
251
	  $tagid='001';
294
    $tagid = '001';
252
}
295
}
253
else {
296
else {
254
   ( $tagid, $subfieldid ) =
297
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
255
            GetMarcFromKohaField( "biblio.biblionumber" );
298
    $tagid ||= "001";
256
	$tagid||="001";
257
}
299
}
258
300
301
my $sth_isbn;
259
# the SQL query to search on isbn
302
# the SQL query to search on isbn
260
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
303
if ($isbn_check) {
304
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
305
}
261
306
262
my $loghandle;
307
my $loghandle;
263
if ($logfile){
308
if ($logfile) {
264
   $loghandle= IO::File->new($logfile, $writemode) ;
309
    $loghandle= IO::File->new($logfile, $writemode);
265
   print $loghandle "id;operation;status\n";
310
    print $loghandle "id;operation;status\n";
266
}
311
}
267
312
313
my $record_number = 0;
268
my $logger = Koha::Logger->get;
314
my $logger = Koha::Logger->get;
269
my $schema = Koha::Database->schema;
315
my $schema = Koha::Database->schema;
270
$schema->txn_begin;
316
$schema->txn_begin;
317
my $marc_records = [];
271
RECORD: while (  ) {
318
RECORD: while (  ) {
272
    my $record;
319
    my $record;
273
    # get records
320
    $record_number++;
321
    # get record
274
    eval { $record = $batch->next() };
322
    eval { $record = $batch->next() };
275
    if ( $@ ) {
323
    if ($@) {
276
        print "Bad MARC record $i: $@ skipped\n";
324
        print "Bad MARC record $record_number: $@ skipped\n";
277
        # FIXME - because MARC::Batch->next() combines grabbing the next
325
        # FIXME - because MARC::Batch->next() combines grabbing the next
278
        # blob and parsing it into one operation, a correctable condition
326
        # blob and parsing it into one operation, a correctable condition
279
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
327
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
Lines 282-560 RECORD: while ( ) { Link Here
282
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
330
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
283
        next;
331
        next;
284
    }
332
    }
285
    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
333
    if ($record) {
286
    last unless ( $record );
334
        # transcode the record to UTF8 if needed & applicable.
287
    $i++;
335
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
288
    if( ($verbose//1)==1 ) { #no dot for verbose==2
336
            my ($guessed_charset, $charset_errors);
289
        print "." . ( $i % 100==0 ? "\n$i" : '' );
337
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
338
            if ($guessed_charset eq 'failed') {
339
                warn "ERROR: failed to perform character conversion for record $record_number\n";
340
                next RECORD;
341
            }
342
        }
343
        SetUTF8Flag($record);
344
        &$localcust($record) if $localcust;
345
        push @{$marc_records}, $record;
346
    }
347
    else {
348
        last;
290
    }
349
    }
350
}
291
351
292
    # transcode the record to UTF8 if needed & applicable.
352
$record_number = 0;
293
    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
353
my $records_total = @{$marc_records};
294
        # FIXME update condition
354
$schema->txn_begin;
295
        my ($guessed_charset, $charset_errors);
355
RECORD: foreach my $record (@{$marc_records}) {
296
         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
356
    $record_number++;
297
        if ($guessed_charset eq 'failed') {
357
    if (($verbose//1) == 1) { #no dot for verbose==2
298
            warn "ERROR: failed to perform character conversion for record $i\n";
358
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
299
            next RECORD;            
300
        }
301
    }
359
    }
302
    SetUTF8Flag($record);
360
303
    if($marc_mod_template_id > 0) {
361
    if ($marc_mod_template_id > 0) {
304
    print "Modifying MARC\n" if $verbose;
362
        print "Modifying MARC\n" if $verbose;
305
    ModifyRecordWithTemplate( $marc_mod_template_id, $record );
363
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
306
    }
364
    }
307
    &$localcust($record) if $localcust;
365
308
    my $isbn;
366
    my $isbn;
309
    # remove trailing - in isbn (only for biblios, of course)
367
    # remove trailing - in isbn (only for biblios, of course)
310
    if( $biblios ) {
368
    if ($biblios && ($cleanisbn || $isbn_check)) {
311
        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
369
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
312
        my $field = $record->field($tag);
370
        my $field = $record->field($tag);
313
        $isbn = $field && $field->subfield('a');
371
        $isbn = $field && $field->subfield('a');
314
        if ( $isbn && $cleanisbn ) {
372
        if ($isbn && $cleanisbn) {
315
            $isbn =~ s/-//g;
373
            $isbn =~ s/-//g;
316
            $field->update('a' => $isbn);
374
            $field->update('a' => $isbn);
317
        }
375
        }
318
    }
376
    }
319
    my $id;
320
    # search for duplicates (based on Local-number)
377
    # search for duplicates (based on Local-number)
321
    my $originalid;
378
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
322
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
379
    my $matched_record_id = undef;
323
    if ($match) {
380
    if ($match) {
324
        require C4::Search;
381
        require C4::Search;
325
        my $query = build_query( $match, $record );
382
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
326
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
383
        my $query = build_query($match, $record);
327
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
384
        $logger->debug("Bulkmarcimport: $query");
385
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
328
        # changed to warn so able to continue with one broken record
386
        # changed to warn so able to continue with one broken record
329
        if ( defined $error ) {
387
        if (defined $error) {
330
            warn "unable to search the database for duplicates : $error";
388
            warn "unable to search the database for duplicates : $error";
331
            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
389
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
332
            next RECORD;
390
            next RECORD;
333
        }
391
        }
334
        if ( $results && scalar(@$results) == 1 ) {
392
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
335
            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
393
        # sub SimpleSearch could return undefined, but only on error, so
336
            SetUTF8Flag($marcrecord);
394
        # should not really need to safeguard here, but do so anyway
337
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
395
        $results //= [];
338
            if ( $authorities && $marcFlavour ) {
396
        if (@{$results} == 1) {
339
                #Skip if authority in database is the same as the on in database
397
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
340
                if ( $marcrecord->field('005') && $record->field('005') &&
398
            SetUTF8Flag($matched_record);
341
                     $marcrecord->field('005')->data && $record->field('005')->data &&
399
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
342
                     $marcrecord->field('005')->data >= $record->field('005')->data ) {
400
401
            if ($authorities && $marc_flavour) {
402
                #Skip if authority in database is the same or newer than the incoming record
403
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
343
                    if ($yamlfile) {
404
                    if ($yamlfile) {
344
                        $yamlhash->{$originalid}->{'authid'} = $id;
405
                        $yamlhash->{$originalid} = YAMLFileEntry(
345
406
                            $matched_record,
346
                        # we recover all subfields of the heading authorities
407
                            $matched_record_id,
347
                        my @subfields;
408
                            0
348
                        foreach my $field ( $marcrecord->field("2..") ) {
409
                        );
349
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
350
                        }
351
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
352
                        $yamlhash->{$originalid}->{'updated'} = 0;
353
                    }
410
                    }
354
                    next;
411
                    next;
355
                }
412
                }
356
            }
413
            }
357
        } elsif ( $results && scalar(@$results) > 1 ) {
358
            $logger->debug("more than one match for $query");
359
        } else {
360
            $logger->debug("nomatch for $query");
361
        }
414
        }
362
    }
415
        elsif(@{$results} > 1) {
363
    if ($keepids && $originalid) {
416
            $logger->debug("More than one match for: $query");
417
        }
418
        else {
419
            $logger->debug("No match for: $query");
420
        }
421
422
        if ($keepids && $originalid) {
364
            my $storeidfield;
423
            my $storeidfield;
365
            if ( length($keepids) == 3 ) {
424
            if (length($keepids) == 3) {
366
                $storeidfield = MARC::Field->new( $keepids, $originalid );
425
                $storeidfield = MARC::Field->new($keepids, $originalid);
367
            } else {
426
            } else {
368
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
427
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
369
            }
428
            }
370
            $record->insert_fields_ordered($storeidfield);
429
            $record->insert_fields_ordered($storeidfield);
371
            $record->delete_field( $record->field($tagid) );
430
            $record->delete_field($record->field($tagid));
431
        }
372
    }
432
    }
433
373
    foreach my $stringfilter (@$filters) {
434
    foreach my $stringfilter (@$filters) {
374
        if ( length($stringfilter) == 3 ) {
435
        if (length($stringfilter) == 3) {
375
            foreach my $field ( $record->field($stringfilter) ) {
436
            foreach my $field ($record->field($stringfilter)) {
376
                $record->delete_field($field);
437
                $record->delete_field($field);
377
                $logger->debug("removed : ", $field->as_string);
438
                $logger->debug("Removed: ", $field->as_string);
378
            }
439
            }
379
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
440
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
380
            my $removetag = $1;
441
            my $removetag = $1;
381
            my $removesubfield = $2;
442
            my $removesubfield = $2;
382
            my $removematch = $3;
443
            my $removematch = $3;
383
            if ( ( $removetag > "010" ) && $removesubfield ) {
444
            if (($removetag > "010") && $removesubfield) {
384
                foreach my $field ( $record->field($removetag) ) {
445
                foreach my $field ($record->field($removetag)) {
385
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
446
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
386
                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));
447
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
387
                }
448
                }
388
            }
449
            }
389
        }
450
        }
390
    }
451
    }
391
    unless ($test_parameter) {
452
    unless ($test_parameter) {
392
        if ($authorities){
453
        if ($authorities) {
393
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
454
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
394
            my $authid= ($id?$id:GuessAuthId($record));
455
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
395
            if ($authid && GetAuthority($authid) && $update ){
456
            if ($authid && GetAuthority($authid) && $update) {
396
            ## Authority has an id and is in database : Replace
457
            ## Authority has an id and is in database : Replace
397
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
458
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
398
                if ($@){
459
                if ($@) {
399
                    warn "Problem with authority $authid Cannot Modify";
460
                    warn "Problem with authority $authid Cannot Modify";
400
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
461
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
462
                }
463
                else{
464
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
401
                }
465
                }
402
				else{
466
            }
403
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
467
            else {
404
				}
405
            }  
406
	        else {
407
            ## True insert in database
468
            ## True insert in database
408
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
469
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
409
                if ($@){
470
                if ($@) {
410
                    warn "Problem with authority $authid Cannot Add".$@;
471
                    warn "Problem with authority $originalid Cannot Add ".$@;
411
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
472
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
412
                }
473
                }
413
   				else{
474
                else {
414
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
475
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
415
				}
476
                }
416
 	        }
477
417
            if ($yamlfile) {
418
            $yamlhash->{$originalid}->{'authid'} = $authid;
419
            my @subfields;
420
            foreach my $field ( $record->field("2..") ) {
421
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
422
            }
478
            }
423
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
479
            if ($yamlfile) {
424
            $yamlhash->{$originalid}->{'updated'} = 1;
480
                $yamlhash->{$originalid} = YAMLFileEntry(
481
                    $record,
482
                    $authid,
483
                    1 #@FIXME: Really always updated?
484
                );
425
            }
485
            }
426
        }
486
        }
427
        else {
487
        else {
428
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
488
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
429
            $biblionumber = $id;
430
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
489
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
431
            if (!$biblionumber && $isbn_check && $isbn) {
490
            if (!$matched_record_id && $isbn_check && $isbn) {
432
    #         warn "search ISBN : $isbn";
433
                $sth_isbn->execute($isbn);
491
                $sth_isbn->execute($isbn);
434
                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
492
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
435
            }
493
            }
436
        	if (defined $idmapfl) {
494
437
			 	if ($sourcetag < "010"){
495
            if (defined $idmapfl && $matched_record_id) {
438
					if ($record->field($sourcetag)){
496
                if ($sourcetag < "010") {
439
					  my $source = $record->field($sourcetag)->data();
497
                    if ($record->field($sourcetag)) {
440
                      printf($idmapfh "%s|%s\n",$source,$biblionumber);
498
                        my $source = $record->field($sourcetag)->data();
441
					}
499
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
442
			    } else {
500
                    }
443
					my $source=$record->subfield($sourcetag,$sourcesubfield);
501
                }
444
                    printf($idmapfh "%s|%s\n",$source,$biblionumber);
502
                else {
445
			  }
503
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
446
			}
504
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
447
					# create biblio, unless we already have it ( either match or isbn )
505
                }
448
            if ($biblionumber) {
506
            }
507
508
            # create biblio, unless we already have it ( either match or isbn )
509
            if ($matched_record_id) {
510
                # TODO: Implement also for authority records!
449
                eval{
511
                eval{
450
                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
512
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
451
                };
513
                };
452
                if ($update) {
514
                if ($update) {
453
                    eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) };
515
                    my $success;
516
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
454
                    if ($@) {
517
                    if ($@) {
455
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
518
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
456
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
519
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
457
                        next RECORD;
520
                        next RECORD;
458
                    } else {
459
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
460
                    }
521
                    }
461
                } else {
522
                    elsif (!$success) {
462
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
523
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
463
                }
524
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
464
            } else {
465
                if ($insert) {
466
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
467
                    if ($@) {
468
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
469
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
470
                        next RECORD;
525
                        next RECORD;
471
                    } else {
472
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
473
                    }
526
                    }
474
                } else {
527
                    else {
475
                    warn "WARNING: Updating record ".($id||$originalid)." failed";
528
                        $record_id = $matched_record_id;
476
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
529
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
530
                    }
531
                }
532
                else {
533
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
534
                }
535
            }
536
            elsif ($insert) {
537
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
538
                if ($@) {
539
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
540
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
477
                    next RECORD;
541
                    next RECORD;
478
                }
542
                }
543
                else {
544
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
545
                }
479
            }
546
            }
480
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
547
            else {
481
            my $error_adding = $@;
548
                warn "WARNING: Updating record ".($originalid)." failed";
482
            # Work on a clone so that if there are real errors, we can maybe
549
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
483
            # fix them up later.
484
			my $clone_record = $record->clone();
485
            C4::Biblio::_strip_item_fields($clone_record, '');
486
            # This sets the marc fields if there was an error, and also calls
487
            # defer_marc_save.
488
            ModBiblioMarc( $clone_record, $biblionumber );
489
            if ( $error_adding ) {
490
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
491
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
492
                # if we failed because of an exception, assume that 
493
                # the MARC columns in biblioitems were not set.
494
                next RECORD;
550
                next RECORD;
495
            }
551
            }
496
 			else{
552
            my $record_has_added_items = 0;
497
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
553
            if ($record_id) {
498
			}
554
                $yamlhash->{$originalid} = $record_id if $yamlfile;
499
            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
555
                # TODO: Add option for skipping items?
500
                # Find the record called 'barcode'
556
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
501
                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
557
                $record_has_added_items = @{$itemnumbers_ref};
502
                # Now remove any items that didn't have a duplicate_barcode error,
558
                my $error_adding = $@;
503
                # erase the barcodes on items that did, and re-add those items.
559
                # Work on a clone so that if there are real errors, we can maybe
504
                my %dupes;
560
                # fix them up later.
505
                foreach my $i (0 .. $#{$errors_ref}) {
561
                my $clone_record = $record->clone();
506
                    my $ref = $errors_ref->[$i];
562
                C4::Biblio::_strip_item_fields($clone_record, $framework);
507
                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
563
                # This sets the marc fields if there was an error, and also calls
508
                        $dupes{$ref->{item_sequence}} = 1;
564
                # defer_marc_save.
509
                        # Delete the error message because we're going to
565
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
510
                        # retry this one.
566
                if ($error_adding) {
511
                        delete $errors_ref->[$i];
567
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
512
                    }
568
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
513
                }
514
                my $seq = 0;
515
                foreach my $field ($record->field($tag)) {
516
                    $seq++;
517
                    if ($dupes{$seq}) {
518
                        # Here we remove the barcode
519
                        $field->delete_subfield(code => $sub);
520
                    } else {
521
                        # otherwise we delete the field because we don't want
522
                        # two of them
523
                        $record->delete_fields($field);
524
                    }
525
                }
526
                # Now re-add the record as before, adding errors to the prev list
527
                my $more_errors;
528
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
529
                if ( $@ ) {
530
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
531
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
532
                    # if we failed because of an exception, assume that
569
                    # if we failed because of an exception, assume that
533
                    # the MARC columns in biblioitems were not set.
570
                    # the MARC columns in biblioitems were not set.
534
                    ModBiblioMarc( $record, $biblionumber );
535
                    next RECORD;
571
                    next RECORD;
536
                } else {
537
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
538
                }
572
                }
539
                push @$errors_ref, @{ $more_errors };
573
                else {
540
            }
574
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
541
            if ($#{ $errors_ref } > -1) {
575
                }
542
                report_item_errors($biblionumber, $errors_ref);
576
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
577
                    # Find the record called 'barcode'
578
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
579
                    # Now remove any items that didn't have a duplicate_barcode error,
580
                    # erase the barcodes on items that did, and re-add those items.
581
                    my %dupes;
582
                    # FIXME: This could cause array out of bounds because shifting down rest of items on array item delete?
583
                    foreach my $i (0 .. $#{$errors_ref}) {
584
                        my $ref = $errors_ref->[$i];
585
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
586
                            $dupes{$ref->{item_sequence}} = 1;
587
                            # Delete the error message because we're going to
588
                            # retry this one.
589
                            delete $errors_ref->[$i];
590
                        }
591
                    }
592
                    my $seq = 0;
593
                    foreach my $field ($record->field($tag)) {
594
                        $seq++;
595
                        if ($dupes{$seq}) {
596
                            # Here we remove the barcode
597
                            $field->delete_subfield(code => $sub);
598
                        }
599
                        else {
600
                            # otherwise we delete the field because we don't want
601
                            # two of them
602
                            $record->delete_fields($field);
603
                        }
604
                    }
605
                    # Now re-add the record as before, adding errors to the prev list
606
                    my $more_errors;
607
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
608
                    $record_has_added_items ||= @{$itemnumbers_ref};
609
                    if ($@) {
610
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
611
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
612
                        # if we failed because of an exception, assume that
613
                        # the MARC columns in biblioitems were not set.
614
615
                        # @FIXME: Why do we save here without stripping items? Besides,
616
                        # save with stripped items has already been performed
617
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
618
                        next RECORD;
619
                    }
620
                    else {
621
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
622
                    }
623
                    push @$errors_ref, @{$more_errors};
624
                }
625
                if (@{$errors_ref}) {
626
                    report_item_errors($record_id, $errors_ref);
627
                }
628
                C4::Biblio::_strip_item_fields($record, $framework);
629
                if ($record_has_added_items || $matched_record_id) {
630
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
631
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
632
                }
633
                push @search_engine_record_ids, $record_id;
634
                push @search_engine_records, $record;
543
            }
635
            }
544
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
545
        }
636
        }
546
        if ( 0 == $i % $commitnum ) {
637
        if ($record_number % $commitnum == 0 || $record_number == $records_total) {
547
            $schema->txn_commit;
638
            $schema->txn_commit;
548
            $schema->txn_begin;
639
            $schema->txn_begin;
640
            if ($indexer) {
641
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
642
                @search_engine_record_ids = ();
643
                @search_engine_records = ();
644
            }
549
        }
645
        }
550
    }
646
    }
551
    print $record->as_formatted()."\n" if ($verbose//0)==2;
647
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
552
    last if $i == $number;
648
    last if $record_number == $number;
553
}
649
}
554
$schema->txn_commit;
650
$schema->txn_commit;
555
651
556
if ($fk_off) {
652
if ($fk_off) {
557
	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
653
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
558
}
654
}
559
655
560
# Restore CataloguingLog and AuthoritiesLog
656
# Restore CataloguingLog and AuthoritiesLog
Lines 562-572 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; Link Here
562
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
658
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
563
659
564
my $timeneeded = gettimeofday - $starttime;
660
my $timeneeded = gettimeofday - $starttime;
565
print "\n$i MARC records done in $timeneeded seconds\n";
661
print "\n$record_number MARC records done in $timeneeded seconds\n";
566
if ($logfile){
662
if ($logfile) {
567
  print $loghandle "file : $input_marc_file\n";
663
    print $loghandle "file : $input_marc_file\n";
568
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
664
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
569
  $loghandle->close;
665
    $loghandle->close;
570
}
666
}
571
if ($yamlfile) {
667
if ($yamlfile) {
572
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
668
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
Lines 574-620 if ($yamlfile) { Link Here
574
}
670
}
575
exit 0;
671
exit 0;
576
672
577
sub GetRecordId{
673
sub YAMLFileEntry {
578
	my $marcrecord=shift;
674
    my ($record, $record_id, $updated) = @_;
579
	my $tag=shift;
675
580
	my $subfield=shift;
676
    my $entry = {
581
	my $id;
677
        authid => $record_id
582
	if ($tag lt "010"){
678
    };
583
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
679
584
	} 
680
    # we recover all subfields of the heading authorities
585
	elsif ($subfield){
681
    my @subfields;
586
		if ($marcrecord->field($tag)){
682
    foreach my $field ($record->field("2..")) {
587
			return $marcrecord->subfield($tag,$subfield);
683
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
588
		}
684
    }
589
	}
685
    $entry->{'subfields'} = \@subfields;
590
	return $id;
686
    $entry->{'updated'} = $updated;
687
688
    return $entry;
591
}
689
}
592
sub build_query {
690
593
	my $match = shift;
691
sub RecordRevisionIsGtOrEq {
594
	my $record=shift;
692
    my ($record_a, $record_b) = @_;
595
        my @searchstrings;
693
    return $record_a->field('005') && $record_b->field('005') &&
596
	foreach my $matchingpoint (@$match){
694
    $record_a->field('005')->data && $record_b->field('005')->data &&
597
	  my $string = build_simplequery($matchingpoint,$record);
695
    $record_a->field('005')->data >= $record_b->field('005')->data;
598
	  push @searchstrings,$string if (length($string)>0);
696
}
697
698
sub GetRecordId {
699
    my $marcrecord = shift;
700
    my $tag = shift;
701
    my $subfield = shift;
702
    if ($tag lt "010") {
703
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
704
    }
705
    elsif ($subfield) {
706
        if ($marcrecord->field($tag)) {
707
            return $marcrecord->subfield($tag, $subfield);
599
        }
708
        }
709
    }
710
    return undef;
711
}
712
sub build_query {
713
    my ($match, $record) = @_;
714
    my @searchstrings;
715
716
    foreach my $matchpoint (@$match) {
717
        my $query = build_simplequery($matchpoint, $record);
718
        push (@searchstrings, $query) if $query;
719
    }
600
    my $op = 'and';
720
    my $op = 'and';
601
    return join(" $op ",@searchstrings);
721
    return join(" $op ", @searchstrings);
602
}
722
}
603
sub build_simplequery {
723
sub build_simplequery {
604
	my $element=shift;
724
    my ($matchpoint, $record) = @_;
605
	my $record=shift;
725
606
    my @searchstrings;
726
    my @searchstrings;
607
    my ($index,$recorddata)=split /,/,$element;
727
    my ($index, $record_data) = split (/,/, $matchpoint);
608
    if ($recorddata=~/(\d{3})(.*)/) {
728
    if ($record_data =~ /(\d{3})(.*)/) {
609
        my ($tag,$subfields) =($1,$2);
729
        my ($tag, $subfields) = ($1, $2);
610
        foreach my $field ($record->field($tag)){
730
        foreach my $field ($record->field($tag)) {
611
		  if (length($field->as_string("$subfields"))>0){
731
            if (length($field->as_string("$subfields")) > 0) {
612
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
732
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
613
		  }
733
            }
614
        }
734
        }
615
    }
735
    }
736
    else {
737
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
738
    }
616
    my $op = 'and';
739
    my $op = 'and';
617
    return join(" $op ",@searchstrings);
740
    return join(" $op ", @searchstrings);
618
}
741
}
619
sub report_item_errors {
742
sub report_item_errors {
620
    my $biblionumber = shift;
743
    my $biblionumber = shift;
Lines 629-648 sub report_item_errors { Link Here
629
        print $msg, "\n";
752
        print $msg, "\n";
630
    }
753
    }
631
}
754
}
632
sub printlog{
755
sub printlog {
633
	my $logelements=shift;
756
    my $logelements = shift;
634
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
757
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
635
}
758
}
636
sub get_heading_fields{
759
sub get_heading_fields {
637
    my $headingfields;
760
    my $headingfields;
638
    if ($authtypes){
761
    if ($authtypes) {
639
        $headingfields = YAML::XS::LoadFile($authtypes);
762
        $headingfields = YAML::XS::LoadFile($authtypes);
640
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
763
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
641
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
764
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
642
    }
765
    }
643
    unless ($headingfields){
766
    unless ($headingfields) {
644
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
767
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
645
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
768
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
646
    }
769
    }
647
    return $headingfields;
770
    return $headingfields;
648
}
771
}
649
- 

Return to bug 29440