View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +28 lines)
Lines 344-349 Unless C<disable_autolink> is passed ModBiblio will relink record headings Link Here
344
to authorities based on settings in the system preferences. This flag allows
344
to authorities based on settings in the system preferences. This flag allows
345
us to not relink records when the authority linker is saving modifications.
345
us to not relink records when the authority linker is saving modifications.
346
346
347
=item C<defer_search_engine_indexing>
348
349
Don't update search index. Useful for bulk updates where this is handled
350
manually for optimization purposes.
351
347
=back
352
=back
348
353
349
Returns 1 on success 0 on failure
354
Returns 1 on success 0 on failure
Lines 353-358 Returns 1 on success 0 on failure Link Here
353
sub ModBiblio {
358
sub ModBiblio {
354
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
359
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
355
    $options //= {};
360
    $options //= {};
361
    my %mod_biblio_marc_options;
362
    $mod_biblio_marc_options{'defer_search_engine_indexing'} =
363
        exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
356
364
357
    if (!$record) {
365
    if (!$record) {
358
        carp 'No record passed to ModBiblio';
366
        carp 'No record passed to ModBiblio';
Lines 416-422 sub ModBiblio { Link Here
416
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
424
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
417
425
418
    # update the MARC record (that now contains biblio and items) with the new record data
426
    # update the MARC record (that now contains biblio and items) with the new record data
419
    ModBiblioMarc( $record, $biblionumber );
427
    ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
420
428
421
    # modify the other koha tables
429
    # modify the other koha tables
422
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
430
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2949-2966 sub _koha_delete_biblio_metadata { Link Here
2949
2957
2950
=head2 ModBiblioMarc
2958
=head2 ModBiblioMarc
2951
2959
2952
  ModBiblioMarc($newrec,$biblionumber);
2960
  ModBiblioMarc($newrec, $biblionumber, $options);
2953
2961
2954
Add MARC XML data for a biblio to koha
2962
Add MARC XML data for a biblio to koha
2955
2963
2956
Function exported, but should NOT be used, unless you really know what you're doing
2964
Function exported, but should NOT be used, unless you really know what you're doing
2957
2965
2966
The C<$options> argument is a hashref with additional parameters:
2967
2968
=over 4
2969
2970
=item C<defer_search_engine_indexing>
2971
2972
Don't update search index. Useful for bulk updates where this is handled
2973
manually for optimization purposes.
2974
2975
=back
2976
2958
=cut
2977
=cut
2959
2978
2960
sub ModBiblioMarc {
2979
sub ModBiblioMarc {
2961
    # pass the MARC::Record to this function, and it will create the records in
2980
    # pass the MARC::Record to this function, and it will create the records in
2962
    # the marcxml field
2981
    # the marcxml field
2963
    my ( $record, $biblionumber ) = @_;
2982
    my ( $record, $biblionumber, $options ) = @_;
2983
    $options //= {};
2984
2964
    if ( !$record ) {
2985
    if ( !$record ) {
2965
        carp 'ModBiblioMarc passed an undefined record';
2986
        carp 'ModBiblioMarc passed an undefined record';
2966
        return;
2987
        return;
Lines 3025-3032 sub ModBiblioMarc { Link Here
3025
    $m_rs->metadata( $record->as_xml_record($encoding) );
3046
    $m_rs->metadata( $record->as_xml_record($encoding) );
3026
    $m_rs->store;
3047
    $m_rs->store;
3027
3048
3028
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3049
    unless (exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'}) {
3029
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3050
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3051
        $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3052
    }
3030
3053
3031
    return $biblionumber;
3054
    return $biblionumber;
3032
}
3055
}
(-)a/misc/migration_tools/bulkmarcimport.pl (-308 / +432 lines)
Lines 23-28 use C4::Biblio qw( Link Here
23
    GetMarcFromKohaField
23
    GetMarcFromKohaField
24
    ModBiblio
24
    ModBiblio
25
    ModBiblioMarc
25
    ModBiblioMarc
26
    GetFrameworkCode
27
    GetMarcBiblio
26
);
28
);
27
use C4::Koha;
29
use C4::Koha;
28
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
30
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
Lines 38-66 use Time::HiRes qw( gettimeofday ); Link Here
38
use Getopt::Long qw( GetOptions );
40
use Getopt::Long qw( GetOptions );
39
use IO::File;
41
use IO::File;
40
use Pod::Usage qw( pod2usage );
42
use Pod::Usage qw( pod2usage );
43
use List::MoreUtils qw(any);
41
44
42
use Koha::Logger;
45
use Koha::Logger;
43
use Koha::Biblios;
46
use Koha::Biblios;
44
use Koha::SearchEngine;
47
use Koha::SearchEngine;
45
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use Koha::Plugins::Handler;
46
50
47
use open qw( :std :encoding(UTF-8) );
51
use open qw( :std :encoding(UTF-8) );
48
binmode( STDOUT, ":encoding(UTF-8)" );
52
binmode(STDOUT, ":encoding(UTF-8)");
49
my ( $input_marc_file, $number, $offset) = ('',0,0);
53
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
50
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
54
my $version;
51
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
55
my $delete;
52
my $cleanisbn = 1;
56
my $test_parameter;
53
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
57
my $skip_marc8_conversion;
58
my $char_encoding;
59
my $verbose;
60
my $commit;
61
my $fk_off;
62
my $format;
63
my $biblios;
64
my $authorities;
65
my $keepids;
66
my $match;
67
my $isbn_check;
68
my $logfile;
69
my $insert;
70
my $filters;
71
my $update;
72
my $all;
73
my $yamlfile;
74
my $authtypes;
75
my $append;
76
my $sourcetag;
77
my $sourcesubfield;
78
my $idmapfl;
79
my $dedup_barcode;
54
my $framework = '';
80
my $framework = '';
55
my $localcust;
81
my $localcust;
56
my $marc_mod_template = '';
82
my $marc_mod_template = '';
57
my $marc_mod_template_id = -1;
83
my $marc_mod_template_id = -1;
58
84
$| = 1;
59
$|=1;
60
85
61
GetOptions(
86
GetOptions(
62
    'commit:f'    => \$commit,
87
    'commit:f' => \$commit,
63
    'file:s'    => \$input_marc_file,
88
    'file:s' => \$input_marc_file,
64
    'n:f' => \$number,
89
    'n:f' => \$number,
65
    'o|offset:f' => \$offset,
90
    'o|offset:f' => \$offset,
66
    'h' => \$version,
91
    'h' => \$version,
Lines 77-98 GetOptions( Link Here
77
    'b|biblios' => \$biblios,
102
    'b|biblios' => \$biblios,
78
    'a|authorities' => \$authorities,
103
    'a|authorities' => \$authorities,
79
    'authtypes:s' => \$authtypes,
104
    'authtypes:s' => \$authtypes,
80
    'filter=s@'     => \$filters,
105
    'filter=s@' => \$filters,
81
    'insert'        => \$insert,
106
    'insert' => \$insert,
82
    'update'        => \$update,
107
    'update' => \$update,
83
    'all'           => \$all,
108
    'all' => \$all,
84
    'match=s@'    => \$match,
109
    'match=s@' => \$match,
85
    'i|isbn' => \$isbn_check,
110
    'i|isbn' => \$isbn_check,
86
    'x:s' => \$sourcetag,
111
    'x:s' => \$sourcetag,
87
    'y:s' => \$sourcesubfield,
112
    'y:s' => \$sourcesubfield,
88
    'idmap:s' => \$idmapfl,
113
    'idmap:s' => \$idmapfl,
89
    'cleanisbn!'     => \$cleanisbn,
114
    'cleanisbn!' => \$cleanisbn,
90
    'yaml:s'        => \$yamlfile,
115
    'yaml:s' => \$yamlfile,
91
    'dedupbarcode' => \$dedup_barcode,
116
    'dedupbarcode' => \$dedup_barcode,
92
    'framework=s' => \$framework,
117
    'framework=s' => \$framework,
93
    'custom:s'    => \$localcust,
118
    'custom:s' => \$localcust,
94
    'marcmodtemplate:s' => \$marc_mod_template,
119
    'marcmodtemplate:s' => \$marc_mod_template,
95
);
120
);
121
96
$biblios ||= !$authorities;
122
$biblios ||= !$authorities;
97
$insert  ||= !$update;
123
$insert  ||= !$update;
98
my $writemode = ($append) ? "a" : "w";
124
my $writemode = ($append) ? "a" : "w";
Lines 104-109 if ($all) { Link Here
104
    $update = 1;
130
    $update = 1;
105
}
131
}
106
132
133
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
134
my $modify_biblio_marc_options = {
135
    defer_search_engine_indexing => $using_elastic_search,
136
    overlay_context => { source => 'bulkmarcimport' }
137
};
138
139
my @search_engine_record_ids;
140
my @search_engine_records;
141
my $indexer;
142
if ($using_elastic_search) {
143
    use Koha::SearchEngine::Elasticsearch::Indexer;
144
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
145
        { index => $authorities ?
146
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
147
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
148
        }
149
    );
150
}
151
107
if ($version || ($input_marc_file eq '')) {
152
if ($version || ($input_marc_file eq '')) {
108
    pod2usage( -verbose => 2 );
153
    pod2usage( -verbose => 2 );
109
    exit;
154
    exit;
Lines 112-124 if( $update && !( $match || $isbn_check ) ) { Link Here
112
    warn "Using -update without -match or -isbn seems to be useless.\n";
157
    warn "Using -update without -match or -isbn seems to be useless.\n";
113
}
158
}
114
159
115
if(defined $localcust) { #local customize module
160
if (defined $localcust) { #local customize module
116
    if(!-e $localcust) {
161
    if (!-e $localcust) {
117
        $localcust= $localcust||'LocalChanges'; #default name
162
        $localcust = $localcust || 'LocalChanges'; #default name
118
        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
163
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
119
        $localcust=~ s/\.pm$//;           #remove extension
164
        $localcust =~ s/\.pm$//;           #remove extension
120
        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
165
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
121
        if(-e $fqcust) {
166
        if (-e $fqcust) {
122
            $localcust= $fqcust;
167
            $localcust= $fqcust;
123
        }
168
        }
124
        else {
169
        else {
Lines 127-133 if(defined $localcust) { #local customize module Link Here
127
        }
172
        }
128
    }
173
    }
129
    require $localcust if $localcust;
174
    require $localcust if $localcust;
130
    $localcust=\&customize if $localcust;
175
    $localcust = \&customize if $localcust;
131
}
176
}
132
177
133
if($marc_mod_template ne '') {
178
if($marc_mod_template ne '') {
Lines 152-219 if($marc_mod_template ne '') { Link Here
152
}
197
}
153
198
154
my $dbh = C4::Context->dbh;
199
my $dbh = C4::Context->dbh;
155
my $heading_fields=get_heading_fields();
200
my $heading_fields = get_heading_fields();
156
157
my $idmapfh;
201
my $idmapfh;
202
158
if (defined $idmapfl) {
203
if (defined $idmapfl) {
159
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
204
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
160
}
205
}
161
206
162
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
207
if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
163
  $sourcetag="910";
208
    $sourcetag = "910";
164
  $sourcesubfield="a";
209
    $sourcesubfield = "a";
165
}
210
}
166
211
167
168
# Disable logging for the biblios and authorities import operation. It would unnecessarily
212
# Disable logging for the biblios and authorities import operation. It would unnecessarily
169
# slow the import
213
# slow the import
170
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
214
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
171
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
215
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
172
216
173
if ($fk_off) {
217
if ($fk_off) {
174
	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
218
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
175
}
219
}
176
220
177
178
if ($delete) {
221
if ($delete) {
179
	if ($biblios){
222
    if ($biblios) {
180
    	print "deleting biblios\n";
223
        print "Deleting biblios\n";
181
        $dbh->do("DELETE FROM biblio");
224
        $dbh->do("DELETE FROM biblio");
182
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
225
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
183
        $dbh->do("DELETE FROM biblioitems");
226
        $dbh->do("DELETE FROM biblioitems");
184
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
227
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
185
        $dbh->do("DELETE FROM items");
228
        $dbh->do("DELETE FROM items");
186
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
229
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
187
	}
230
    }
188
	else {
231
    else {
189
    	print "deleting authorities\n";
232
        print "Deleting authorities\n";
190
    	$dbh->do("truncate auth_header");
233
        $dbh->do("truncate auth_header");
191
	}
234
    }
192
    $dbh->do("truncate zebraqueue");
235
    $dbh->do("truncate zebraqueue");
193
}
236
}
194
237
195
196
197
if ($test_parameter) {
238
if ($test_parameter) {
198
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
239
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
199
}
240
}
200
241
201
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
242
my $batch;
243
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
202
244
203
# The definition of $searcher must be before MARC::Batch->new
245
# The definition of $searcher must be before MARC::Batch->new
204
my $searcher = Koha::SearchEngine::Search->new(
246
my $searcher = Koha::SearchEngine::Search->new(
205
    {
247
    {
206
        index => (
248
        index => (
207
              $authorities
249
            $authorities
208
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
250
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
209
            : $Koha::SearchEngine::BIBLIOS_INDEX
251
            : $Koha::SearchEngine::BIBLIOS_INDEX
210
        )
252
        )
211
    }
253
    }
212
);
254
);
213
255
214
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
256
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
215
my $starttime = gettimeofday;
257
my $starttime = gettimeofday;
216
my $batch;
258
217
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
259
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
218
if (defined $format && $format =~ /XML/i) {
260
if (defined $format && $format =~ /XML/i) {
219
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
261
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
Lines 226-279 if (defined $format && $format =~ /XML/i) { Link Here
226
    #       extract the records, not using regexes to look
268
    #       extract the records, not using regexes to look
227
    #       for <record>.*</record>.
269
    #       for <record>.*</record>.
228
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
270
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
229
    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
271
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
230
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
272
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
231
    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
273
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
232
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
274
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
233
    $batch = MARC::Batch->new( 'XML', $fh );
275
    $batch = MARC::Batch->new('XML', $fh);
234
} else {
276
}
235
    $batch = MARC::Batch->new( 'USMARC', $fh );
277
else {
278
    $batch = MARC::Batch->new('USMARC', $fh);
236
}
279
}
280
237
$batch->warnings_off();
281
$batch->warnings_off();
238
$batch->strict_off();
282
$batch->strict_off();
239
my $i=0;
240
my $commitnum = $commit ? $commit : 50;
283
my $commitnum = $commit ? $commit : 50;
241
my $yamlhash;
284
my $yamlhash;
242
285
243
# Skip file offset
286
# Skip file offset
244
if ( $offset ) {
287
if ($offset) {
245
    print "Skipping file offset: $offset records\n";
288
    print "Skipping file offset: $offset records\n";
246
    $batch->next() while ($offset--);
289
    $batch->next() while ($offset--);
247
}
290
}
248
291
249
my ($tagid,$subfieldid);
292
my ($tagid, $subfieldid);
250
if ($authorities){
293
if ($authorities) {
251
	  $tagid='001';
294
    $tagid = '001';
252
}
295
}
253
else {
296
else {
254
   ( $tagid, $subfieldid ) =
297
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
255
            GetMarcFromKohaField( "biblio.biblionumber" );
298
    $tagid ||= "001";
256
	$tagid||="001";
257
}
299
}
258
300
301
my $sth_isbn;
259
# the SQL query to search on isbn
302
# the SQL query to search on isbn
260
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
303
if ($isbn_check) {
304
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
305
}
261
306
262
my $loghandle;
307
my $loghandle;
263
if ($logfile){
308
if ($logfile) {
264
   $loghandle= IO::File->new($logfile, $writemode) ;
309
    $loghandle= IO::File->new($logfile, $writemode);
265
   print $loghandle "id;operation;status\n";
310
    print $loghandle "id;operation;status\n";
266
}
311
}
267
312
313
my $record_number = 0;
268
my $logger = Koha::Logger->get;
314
my $logger = Koha::Logger->get;
269
my $schema = Koha::Database->schema;
315
my $schema = Koha::Database->schema;
270
$schema->txn_begin;
316
$schema->txn_begin;
317
my $marc_records = [];
271
RECORD: while (  ) {
318
RECORD: while (  ) {
272
    my $record;
319
    my $record;
273
    # get records
320
    $record_number++;
321
    # get record
274
    eval { $record = $batch->next() };
322
    eval { $record = $batch->next() };
275
    if ( $@ ) {
323
    if ($@) {
276
        print "Bad MARC record $i: $@ skipped\n";
324
        print "Bad MARC record $record_number: $@ skipped\n";
277
        # FIXME - because MARC::Batch->next() combines grabbing the next
325
        # FIXME - because MARC::Batch->next() combines grabbing the next
278
        # blob and parsing it into one operation, a correctable condition
326
        # blob and parsing it into one operation, a correctable condition
279
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
327
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
Lines 282-560 RECORD: while ( ) { Link Here
282
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
330
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
283
        next;
331
        next;
284
    }
332
    }
285
    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
333
    if ($record) {
286
    last unless ( $record );
334
        # transcode the record to UTF8 if needed & applicable.
287
    $i++;
335
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
288
    if( ($verbose//1)==1 ) { #no dot for verbose==2
336
            my ($guessed_charset, $charset_errors);
289
        print "." . ( $i % 100==0 ? "\n$i" : '' );
337
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
338
            if ($guessed_charset eq 'failed') {
339
                warn "ERROR: failed to perform character conversion for record $record_number\n";
340
                next RECORD;
341
            }
342
        }
343
        SetUTF8Flag($record);
344
        &$localcust($record) if $localcust;
345
        push @{$marc_records}, $record;
346
    }
347
    else {
348
        last;
290
    }
349
    }
350
}
291
351
292
    # transcode the record to UTF8 if needed & applicable.
352
$record_number = 0;
293
    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
353
my $records_total = @{$marc_records};
294
        # FIXME update condition
354
$schema->txn_begin;
295
        my ($guessed_charset, $charset_errors);
355
RECORD: foreach my $record (@{$marc_records}) {
296
         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
356
    $record_number++;
297
        if ($guessed_charset eq 'failed') {
357
    if (($verbose//1) == 1) { #no dot for verbose==2
298
            warn "ERROR: failed to perform character conversion for record $i\n";
358
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
299
            next RECORD;            
300
        }
301
    }
359
    }
302
    SetUTF8Flag($record);
360
303
    if($marc_mod_template_id > 0) {
361
    if ($marc_mod_template_id > 0) {
304
    print "Modifying MARC\n" if $verbose;
362
        print "Modifying MARC\n" if $verbose;
305
    ModifyRecordWithTemplate( $marc_mod_template_id, $record );
363
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
306
    }
364
    }
307
    &$localcust($record) if $localcust;
365
308
    my $isbn;
309
    # remove trailing - in isbn (only for biblios, of course)
366
    # remove trailing - in isbn (only for biblios, of course)
310
    if( $biblios ) {
367
    if ($biblios) {
311
        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
368
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
312
        my $field = $record->field($tag);
369
        my $field = $record->field($tag);
313
        $isbn = $field && $field->subfield('a');
370
        my $isbn = $field && $field->subfield('a');
314
        if ( $isbn && $cleanisbn ) {
371
        if ( $isbn && $cleanisbn ) {
315
            $isbn =~ s/-//g;
372
            $isbn =~ s/-//g;
316
            $field->update('a' => $isbn);
373
            $field->update('a' => $isbn);
317
        }
374
        }
318
    }
375
    }
319
    my $id;
320
    # search for duplicates (based on Local-number)
376
    # search for duplicates (based on Local-number)
321
    my $originalid;
377
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
322
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
378
    my $matched_record_id = undef;
323
    if ($match) {
379
    if ($match) {
324
        require C4::Search;
380
        require C4::Search;
325
        my $query = build_query( $match, $record );
381
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
326
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
382
        my $query = build_query($match, $record);
327
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
383
        $logger->debug("Bulkmarcimport: $query");
384
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
328
        # changed to warn so able to continue with one broken record
385
        # changed to warn so able to continue with one broken record
329
        if ( defined $error ) {
386
        if (defined $error) {
330
            warn "unable to search the database for duplicates : $error";
387
            warn "unable to search the database for duplicates : $error";
331
            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
388
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
332
            next RECORD;
389
            next RECORD;
333
        }
390
        }
334
        if ( $results && scalar(@$results) == 1 ) {
391
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
335
            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
392
        # sub SimpleSearch could return undefined, but only on error, so
336
            SetUTF8Flag($marcrecord);
393
        # should not really need to safeguard here, but do so anyway
337
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
394
        $results //= [];
338
            if ( $authorities && $marcFlavour ) {
395
        if (@{$results} == 1) {
339
                #Skip if authority in database is the same as the on in database
396
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
340
                if ( $marcrecord->field('005') && $record->field('005') &&
397
            SetUTF8Flag($matched_record);
341
                     $marcrecord->field('005')->data && $record->field('005')->data &&
398
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
342
                     $marcrecord->field('005')->data >= $record->field('005')->data ) {
399
400
            if ($authorities && $marc_flavour) {
401
                #Skip if authority in database is the same or newer than the incoming record
402
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
343
                    if ($yamlfile) {
403
                    if ($yamlfile) {
344
                        $yamlhash->{$originalid}->{'authid'} = $id;
404
                        $yamlhash->{$originalid} = YAMLFileEntry(
345
405
                            $matched_record,
346
                        # we recover all subfields of the heading authorities
406
                            $matched_record_id,
347
                        my @subfields;
407
                            0
348
                        foreach my $field ( $marcrecord->field("2..") ) {
408
                        );
349
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
350
                        }
351
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
352
                        $yamlhash->{$originalid}->{'updated'} = 0;
353
                    }
409
                    }
354
                    next;
410
                    next;
355
                }
411
                }
356
            }
412
            }
357
        } elsif ( $results && scalar(@$results) > 1 ) {
358
            $logger->debug("more than one match for $query");
359
        } else {
360
            $logger->debug("nomatch for $query");
361
        }
413
        }
362
    }
414
        elsif(@{$results} > 1) {
363
    if ($keepids && $originalid) {
415
            $logger->debug("More than one match for: $query");
416
        }
417
        else {
418
            $logger->debug("No match for: $query");
419
        }
420
421
        if ($keepids && $originalid) {
364
            my $storeidfield;
422
            my $storeidfield;
365
            if ( length($keepids) == 3 ) {
423
            if (length($keepids) == 3) {
366
                $storeidfield = MARC::Field->new( $keepids, $originalid );
424
                $storeidfield = MARC::Field->new($keepids, $originalid);
367
            } else {
425
            } else {
368
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
426
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
369
            }
427
            }
370
            $record->insert_fields_ordered($storeidfield);
428
            $record->insert_fields_ordered($storeidfield);
371
            $record->delete_field( $record->field($tagid) );
429
            $record->delete_field($record->field($tagid));
430
        }
372
    }
431
    }
432
373
    foreach my $stringfilter (@$filters) {
433
    foreach my $stringfilter (@$filters) {
374
        if ( length($stringfilter) == 3 ) {
434
        if (length($stringfilter) == 3) {
375
            foreach my $field ( $record->field($stringfilter) ) {
435
            foreach my $field ($record->field($stringfilter)) {
376
                $record->delete_field($field);
436
                $record->delete_field($field);
377
                $logger->debug("removed : ", $field->as_string);
437
                $logger->debug("Removed: ", $field->as_string);
378
            }
438
            }
379
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
439
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
380
            my $removetag = $1;
440
            my $removetag = $1;
381
            my $removesubfield = $2;
441
            my $removesubfield = $2;
382
            my $removematch = $3;
442
            my $removematch = $3;
383
            if ( ( $removetag > "010" ) && $removesubfield ) {
443
            if (($removetag > "010") && $removesubfield) {
384
                foreach my $field ( $record->field($removetag) ) {
444
                foreach my $field ($record->field($removetag)) {
385
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
445
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
386
                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));
446
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
387
                }
447
                }
388
            }
448
            }
389
        }
449
        }
390
    }
450
    }
391
    unless ($test_parameter) {
451
    unless ($test_parameter) {
392
        if ($authorities){
452
        if ($authorities) {
393
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
453
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
394
            my $authid= ($id?$id:GuessAuthId($record));
454
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
395
            if ($authid && GetAuthority($authid) && $update ){
455
            if ($authid && GetAuthority($authid) && $update) {
396
            ## Authority has an id and is in database : Replace
456
            ## Authority has an id and is in database : Replace
397
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
457
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
398
                if ($@){
458
                if ($@) {
399
                    warn "Problem with authority $authid Cannot Modify";
459
                    warn "Problem with authority $authid Cannot Modify";
400
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
460
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
461
                }
462
                else{
463
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
401
                }
464
                }
402
				else{
465
            }
403
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
466
            else {
404
				}
405
            }  
406
	        else {
407
            ## True insert in database
467
            ## True insert in database
408
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
468
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
409
                if ($@){
469
                if ($@) {
410
                    warn "Problem with authority $authid Cannot Add".$@;
470
                    warn "Problem with authority $originalid Cannot Add ".$@;
411
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
471
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
412
                }
472
                }
413
   				else{
473
                else {
414
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
474
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
415
				}
475
                }
416
 	        }
476
417
            if ($yamlfile) {
418
            $yamlhash->{$originalid}->{'authid'} = $authid;
419
            my @subfields;
420
            foreach my $field ( $record->field("2..") ) {
421
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
422
            }
477
            }
423
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
478
            if ($yamlfile) {
424
            $yamlhash->{$originalid}->{'updated'} = 1;
479
                $yamlhash->{$originalid} = YAMLFileEntry(
480
                    $record,
481
                    $authid,
482
                    1 #@FIXME: Really always updated?
483
                );
425
            }
484
            }
426
        }
485
        }
427
        else {
486
        else {
428
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
487
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
429
            $biblionumber = $id;
430
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
488
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
431
            if (!$biblionumber && $isbn_check && $isbn) {
489
            if (!$matched_record_id && $isbn_check) {
432
    #         warn "search ISBN : $isbn";
490
                my $field = $record->field($marc_flavour eq 'UNIMARC' ? '010' : '020');
491
                my $isbn = $field && $field->subfield('a');
492
                # TODO: Check that isbn has valid valiue, or $field->subfield('a')->data??
433
                $sth_isbn->execute($isbn);
493
                $sth_isbn->execute($isbn);
434
                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
494
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
435
            }
495
            }
436
        	if (defined $idmapfl) {
496
437
			 	if ($sourcetag < "010"){
497
            if (defined $idmapfl && $matched_record_id) {
438
					if ($record->field($sourcetag)){
498
                if ($sourcetag < "010") {
439
					  my $source = $record->field($sourcetag)->data();
499
                    if ($record->field($sourcetag)) {
440
                      printf($idmapfh "%s|%s\n",$source,$biblionumber);
500
                        my $source = $record->field($sourcetag)->data();
441
					}
501
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
442
			    } else {
502
                    }
443
					my $source=$record->subfield($sourcetag,$sourcesubfield);
503
                }
444
                    printf($idmapfh "%s|%s\n",$source,$biblionumber);
504
                else {
445
			  }
505
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
446
			}
506
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
447
					# create biblio, unless we already have it ( either match or isbn )
507
                }
448
            if ($biblionumber) {
508
            }
509
510
            # create biblio, unless we already have it ( either match or isbn )
511
            if ($matched_record_id) {
512
                # TODO: Implement also for authority records!
449
                eval{
513
                eval{
450
                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
514
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
451
                };
515
                };
452
                if ($update) {
516
                if ($update) {
453
                    eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) };
517
                    my $success;
518
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
454
                    if ($@) {
519
                    if ($@) {
455
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
520
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
456
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
521
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
457
                        next RECORD;
522
                        next RECORD;
458
                    } else {
459
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
460
                    }
523
                    }
461
                } else {
524
                    elsif (!$success) {
462
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
525
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
463
                }
526
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
464
            } else {
465
                if ($insert) {
466
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
467
                    if ($@) {
468
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
469
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
470
                        next RECORD;
527
                        next RECORD;
471
                    } else {
472
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
473
                    }
528
                    }
474
                } else {
529
                    else {
475
                    warn "WARNING: Updating record ".($id||$originalid)." failed";
530
                        $record_id = $matched_record_id;
476
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
531
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
532
                    }
533
                }
534
                else {
535
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
536
                }
537
            }
538
            elsif ($insert) {
539
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
540
                if ($@) {
541
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
542
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
477
                    next RECORD;
543
                    next RECORD;
478
                }
544
                }
545
                else {
546
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
547
                }
479
            }
548
            }
480
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
549
            else {
481
            my $error_adding = $@;
550
                warn "WARNING: Updating record ".($originalid)." failed";
482
            # Work on a clone so that if there are real errors, we can maybe
551
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
483
            # fix them up later.
484
			my $clone_record = $record->clone();
485
            C4::Biblio::_strip_item_fields($clone_record, '');
486
            # This sets the marc fields if there was an error, and also calls
487
            # defer_marc_save.
488
            ModBiblioMarc( $clone_record, $biblionumber );
489
            if ( $error_adding ) {
490
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
491
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
492
                # if we failed because of an exception, assume that 
493
                # the MARC columns in biblioitems were not set.
494
                next RECORD;
552
                next RECORD;
495
            }
553
            }
496
 			else{
554
            my $record_has_added_items = 0;
497
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
555
            if ($record_id) {
498
			}
556
                $yamlhash->{$originalid} = $record_id if $yamlfile;
499
            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
557
                # TODO: Add option for skipping items?
500
                # Find the record called 'barcode'
558
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
501
                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
559
                $record_has_added_items = @{$itemnumbers_ref};
502
                # Now remove any items that didn't have a duplicate_barcode error,
560
                my $error_adding = $@;
503
                # erase the barcodes on items that did, and re-add those items.
561
                # Work on a clone so that if there are real errors, we can maybe
504
                my %dupes;
562
                # fix them up later.
505
                foreach my $i (0 .. $#{$errors_ref}) {
563
                my $clone_record = $record->clone();
506
                    my $ref = $errors_ref->[$i];
564
                C4::Biblio::_strip_item_fields($clone_record, $framework);
507
                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
565
                # This sets the marc fields if there was an error, and also calls
508
                        $dupes{$ref->{item_sequence}} = 1;
566
                # defer_marc_save.
509
                        # Delete the error message because we're going to
567
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
510
                        # retry this one.
568
                if ($error_adding) {
511
                        delete $errors_ref->[$i];
569
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
512
                    }
570
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
513
                }
514
                my $seq = 0;
515
                foreach my $field ($record->field($tag)) {
516
                    $seq++;
517
                    if ($dupes{$seq}) {
518
                        # Here we remove the barcode
519
                        $field->delete_subfield(code => $sub);
520
                    } else {
521
                        # otherwise we delete the field because we don't want
522
                        # two of them
523
                        $record->delete_fields($field);
524
                    }
525
                }
526
                # Now re-add the record as before, adding errors to the prev list
527
                my $more_errors;
528
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
529
                if ( $@ ) {
530
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
531
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
532
                    # if we failed because of an exception, assume that
571
                    # if we failed because of an exception, assume that
533
                    # the MARC columns in biblioitems were not set.
572
                    # the MARC columns in biblioitems were not set.
534
                    ModBiblioMarc( $record, $biblionumber );
535
                    next RECORD;
573
                    next RECORD;
536
                } else {
537
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
538
                }
574
                }
539
                push @$errors_ref, @{ $more_errors };
575
                else {
540
            }
576
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
541
            if ($#{ $errors_ref } > -1) {
577
                }
542
                report_item_errors($biblionumber, $errors_ref);
578
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
579
                    # Find the record called 'barcode'
580
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
581
                    # Now remove any items that didn't have a duplicate_barcode error,
582
                    # erase the barcodes on items that did, and re-add those items.
583
                    my %dupes;
584
                    # FIXME: This could cause array out of bounds because shifting down rest of items on array item delete?
585
                    foreach my $i (0 .. $#{$errors_ref}) {
586
                        my $ref = $errors_ref->[$i];
587
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
588
                            $dupes{$ref->{item_sequence}} = 1;
589
                            # Delete the error message because we're going to
590
                            # retry this one.
591
                            delete $errors_ref->[$i];
592
                        }
593
                    }
594
                    my $seq = 0;
595
                    foreach my $field ($record->field($tag)) {
596
                        $seq++;
597
                        if ($dupes{$seq}) {
598
                            # Here we remove the barcode
599
                            $field->delete_subfield(code => $sub);
600
                        }
601
                        else {
602
                            # otherwise we delete the field because we don't want
603
                            # two of them
604
                            $record->delete_fields($field);
605
                        }
606
                    }
607
                    # Now re-add the record as before, adding errors to the prev list
608
                    my $more_errors;
609
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
610
                    $record_has_added_items ||= @{$itemnumbers_ref};
611
                    if ($@) {
612
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
613
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
614
                        # if we failed because of an exception, assume that
615
                        # the MARC columns in biblioitems were not set.
616
617
                        # @FIXME: Why do we save here without stripping items? Besides,
618
                        # save with stripped items has already been performed
619
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
620
                        next RECORD;
621
                    }
622
                    else {
623
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
624
                    }
625
                    push @$errors_ref, @{$more_errors};
626
                }
627
                if (@{$errors_ref}) {
628
                    report_item_errors($record_id, $errors_ref);
629
                }
630
                C4::Biblio::_strip_item_fields($record, $framework);
631
                if ($record_has_added_items || $matched_record_id) {
632
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
633
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
634
                }
635
                push @search_engine_record_ids, $record_id;
636
                push @search_engine_records, $record;
543
            }
637
            }
544
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
545
        }
638
        }
546
        if ( 0 == $i % $commitnum ) {
639
        if ($record_number % $commitnum == 0 || $record_number == $records_total) {
547
            $schema->txn_commit;
640
            $schema->txn_commit;
548
            $schema->txn_begin;
641
            $schema->txn_begin;
642
            if ($indexer) {
643
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
644
                @search_engine_record_ids = ();
645
                @search_engine_records = ();
646
            }
549
        }
647
        }
550
    }
648
    }
551
    print $record->as_formatted()."\n" if ($verbose//0)==2;
649
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
552
    last if $i == $number;
650
    last if $record_number == $number;
553
}
651
}
554
$schema->txn_commit;
652
$schema->txn_commit;
555
653
556
if ($fk_off) {
654
if ($fk_off) {
557
	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
655
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
558
}
656
}
559
657
560
# Restore CataloguingLog and AuthoritiesLog
658
# Restore CataloguingLog and AuthoritiesLog
Lines 562-572 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; Link Here
562
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
660
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
563
661
564
my $timeneeded = gettimeofday - $starttime;
662
my $timeneeded = gettimeofday - $starttime;
565
print "\n$i MARC records done in $timeneeded seconds\n";
663
print "\n$record_number MARC records done in $timeneeded seconds\n";
566
if ($logfile){
664
if ($logfile) {
567
  print $loghandle "file : $input_marc_file\n";
665
    print $loghandle "file : $input_marc_file\n";
568
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
666
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
569
  $loghandle->close;
667
    $loghandle->close;
570
}
668
}
571
if ($yamlfile) {
669
if ($yamlfile) {
572
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
670
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
Lines 574-620 if ($yamlfile) { Link Here
574
}
672
}
575
exit 0;
673
exit 0;
576
674
577
sub GetRecordId{
675
sub YAMLFileEntry {
578
	my $marcrecord=shift;
676
    my ($record, $record_id, $updated) = @_;
579
	my $tag=shift;
677
580
	my $subfield=shift;
678
    my $entry = {
581
	my $id;
679
        authid => $record_id
582
	if ($tag lt "010"){
680
    };
583
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
681
584
	} 
682
    # we recover all subfields of the heading authorities
585
	elsif ($subfield){
683
    my @subfields;
586
		if ($marcrecord->field($tag)){
684
    foreach my $field ($record->field("2..")) {
587
			return $marcrecord->subfield($tag,$subfield);
685
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
588
		}
686
    }
589
	}
687
    $entry->{'subfields'} = \@subfields;
590
	return $id;
688
    $entry->{'updated'} = $updated;
689
690
    return $entry;
591
}
691
}
592
sub build_query {
692
593
	my $match = shift;
693
sub RecordRevisionIsGtOrEq {
594
	my $record=shift;
694
    my ($record_a, $record_b) = @_;
595
        my @searchstrings;
695
    return $record_a->field('005') && $record_b->field('005') &&
596
	foreach my $matchingpoint (@$match){
696
    $record_a->field('005')->data && $record_b->field('005')->data &&
597
	  my $string = build_simplequery($matchingpoint,$record);
697
    $record_a->field('005')->data >= $record_b->field('005')->data;
598
	  push @searchstrings,$string if (length($string)>0);
698
}
699
700
sub GetRecordId {
701
    my $marcrecord = shift;
702
    my $tag = shift;
703
    my $subfield = shift;
704
    if ($tag lt "010") {
705
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
706
    }
707
    elsif ($subfield) {
708
        if ($marcrecord->field($tag)) {
709
            return $marcrecord->subfield($tag, $subfield);
599
        }
710
        }
711
    }
712
    return undef;
713
}
714
sub build_query {
715
    my ($match, $record) = @_;
716
    my @searchstrings;
717
718
    foreach my $matchpoint (@$match) {
719
        my $query = build_simplequery($matchpoint, $record);
720
        push (@searchstrings, $query) if $query;
721
    }
600
    my $op = 'and';
722
    my $op = 'and';
601
    return join(" $op ",@searchstrings);
723
    return join(" $op ", @searchstrings);
602
}
724
}
603
sub build_simplequery {
725
sub build_simplequery {
604
	my $element=shift;
726
    my ($matchpoint, $record) = @_;
605
	my $record=shift;
727
606
    my @searchstrings;
728
    my @searchstrings;
607
    my ($index,$recorddata)=split /,/,$element;
729
    my ($index, $record_data) = split (/,/, $matchpoint);
608
    if ($recorddata=~/(\d{3})(.*)/) {
730
    if ($record_data =~ /(\d{3})(.*)/) {
609
        my ($tag,$subfields) =($1,$2);
731
        my ($tag, $subfields) = ($1, $2);
610
        foreach my $field ($record->field($tag)){
732
        foreach my $field ($record->field($tag)) {
611
		  if (length($field->as_string("$subfields"))>0){
733
            if (length($field->as_string("$subfields")) > 0) {
612
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
734
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
613
		  }
735
            }
614
        }
736
        }
615
    }
737
    }
738
    else {
739
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
740
    }
616
    my $op = 'and';
741
    my $op = 'and';
617
    return join(" $op ",@searchstrings);
742
    return join(" $op ", @searchstrings);
618
}
743
}
619
sub report_item_errors {
744
sub report_item_errors {
620
    my $biblionumber = shift;
745
    my $biblionumber = shift;
Lines 629-648 sub report_item_errors { Link Here
629
        print $msg, "\n";
754
        print $msg, "\n";
630
    }
755
    }
631
}
756
}
632
sub printlog{
757
sub printlog {
633
	my $logelements=shift;
758
    my $logelements = shift;
634
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
759
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
635
}
760
}
636
sub get_heading_fields{
761
sub get_heading_fields {
637
    my $headingfields;
762
    my $headingfields;
638
    if ($authtypes){
763
    if ($authtypes) {
639
        $headingfields = YAML::XS::LoadFile($authtypes);
764
        $headingfields = YAML::XS::LoadFile($authtypes);
640
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
765
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
641
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
766
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
642
    }
767
    }
643
    unless ($headingfields){
768
    unless ($headingfields) {
644
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
769
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
645
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
770
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
646
    }
771
    }
647
    return $headingfields;
772
    return $headingfields;
648
}
773
}
649
- 

Return to bug 29440