View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +28 lines)
Lines 344-349 Unless C<disable_autolink> is passed ModBiblio will relink record headings Link Here
344
to authorities based on settings in the system preferences. This flag allows
344
to authorities based on settings in the system preferences. This flag allows
345
us to not relink records when the authority linker is saving modifications.
345
us to not relink records when the authority linker is saving modifications.
346
346
347
=item C<defer_search_engine_indexing>
348
349
Don't update search index. Useful for bulk updates where this is handled
350
manually for optimization purposes.
351
347
=back
352
=back
348
353
349
Returns 1 on success 0 on failure
354
Returns 1 on success 0 on failure
Lines 353-358 Returns 1 on success 0 on failure Link Here
353
sub ModBiblio {
358
sub ModBiblio {
354
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
359
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
355
    $options //= {};
360
    $options //= {};
361
    my %mod_biblio_marc_options;
362
    $mod_biblio_marc_options{'defer_search_engine_indexing'} =
363
        exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
356
364
357
    if (!$record) {
365
    if (!$record) {
358
        carp 'No record passed to ModBiblio';
366
        carp 'No record passed to ModBiblio';
Lines 416-422 sub ModBiblio { Link Here
416
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
424
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
417
425
418
    # update the MARC record (that now contains biblio and items) with the new record data
426
    # update the MARC record (that now contains biblio and items) with the new record data
419
    ModBiblioMarc( $record, $biblionumber );
427
    ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
420
428
421
    # modify the other koha tables
429
    # modify the other koha tables
422
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
430
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2949-2966 sub _koha_delete_biblio_metadata { Link Here
2949
2957
2950
=head2 ModBiblioMarc
2958
=head2 ModBiblioMarc
2951
2959
2952
  ModBiblioMarc($newrec,$biblionumber);
2960
  ModBiblioMarc($newrec, $biblionumber, $options);
2953
2961
2954
Add MARC XML data for a biblio to koha
2962
Add MARC XML data for a biblio to koha
2955
2963
2956
Function exported, but should NOT be used, unless you really know what you're doing
2964
Function exported, but should NOT be used, unless you really know what you're doing
2957
2965
2966
The C<$options> argument is a hashref with additional parameters:
2967
2968
=over 4
2969
2970
=item C<defer_search_engine_indexing>
2971
2972
Don't update search index. Useful for bulk updates where this is handled
2973
manually for optimization purposes.
2974
2975
=back
2976
2958
=cut
2977
=cut
2959
2978
2960
sub ModBiblioMarc {
2979
sub ModBiblioMarc {
2961
    # pass the MARC::Record to this function, and it will create the records in
2980
    # pass the MARC::Record to this function, and it will create the records in
2962
    # the marcxml field
2981
    # the marcxml field
2963
    my ( $record, $biblionumber ) = @_;
2982
    my ( $record, $biblionumber, $options ) = @_;
2983
    $options //= {};
2984
2964
    if ( !$record ) {
2985
    if ( !$record ) {
2965
        carp 'ModBiblioMarc passed an undefined record';
2986
        carp 'ModBiblioMarc passed an undefined record';
2966
        return;
2987
        return;
Lines 3025-3032 sub ModBiblioMarc { Link Here
3025
    $m_rs->metadata( $record->as_xml_record($encoding) );
3046
    $m_rs->metadata( $record->as_xml_record($encoding) );
3026
    $m_rs->store;
3047
    $m_rs->store;
3027
3048
3028
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3049
    unless (exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'}) {
3029
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3050
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
3051
        $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3052
    }
3030
3053
3031
    return $biblionumber;
3054
    return $biblionumber;
3032
}
3055
}
(-)a/misc/migration_tools/bulkmarcimport.pl (-308 / +425 lines)
Lines 23-28 use C4::Biblio qw( Link Here
23
    GetMarcFromKohaField
23
    GetMarcFromKohaField
24
    ModBiblio
24
    ModBiblio
25
    ModBiblioMarc
25
    ModBiblioMarc
26
    GetFrameworkCode
27
    GetMarcBiblio
26
);
28
);
27
use C4::Koha;
29
use C4::Koha;
28
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
30
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
Lines 45-66 use Koha::SearchEngine; Link Here
45
use Koha::SearchEngine::Search;
47
use Koha::SearchEngine::Search;
46
48
47
use open qw( :std :encoding(UTF-8) );
49
use open qw( :std :encoding(UTF-8) );
48
binmode( STDOUT, ":encoding(UTF-8)" );
50
binmode(STDOUT, ":encoding(UTF-8)");
49
my ( $input_marc_file, $number, $offset) = ('',0,0);
51
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
50
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
52
my $version;
51
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
53
my $delete;
52
my $cleanisbn = 1;
54
my $test_parameter;
53
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
55
my $skip_marc8_conversion;
56
my $char_encoding;
57
my $verbose;
58
my $commit;
59
my $fk_off;
60
my $format;
61
my $biblios;
62
my $authorities;
63
my $keepids;
64
my $match;
65
my $isbn_check;
66
my $logfile;
67
my $insert;
68
my $filters;
69
my $update;
70
my $all;
71
my $yamlfile;
72
my $authtypes;
73
my $append;
74
my $sourcetag;
75
my $sourcesubfield;
76
my $idmapfl;
77
my $dedup_barcode;
54
my $framework = '';
78
my $framework = '';
55
my $localcust;
79
my $localcust;
56
my $marc_mod_template = '';
80
my $marc_mod_template = '';
57
my $marc_mod_template_id = -1;
81
my $marc_mod_template_id = -1;
58
82
$| = 1;
59
$|=1;
60
83
61
GetOptions(
84
GetOptions(
62
    'commit:f'    => \$commit,
85
    'commit:f' => \$commit,
63
    'file:s'    => \$input_marc_file,
86
    'file:s' => \$input_marc_file,
64
    'n:f' => \$number,
87
    'n:f' => \$number,
65
    'o|offset:f' => \$offset,
88
    'o|offset:f' => \$offset,
66
    'h' => \$version,
89
    'h' => \$version,
Lines 77-98 GetOptions( Link Here
77
    'b|biblios' => \$biblios,
100
    'b|biblios' => \$biblios,
78
    'a|authorities' => \$authorities,
101
    'a|authorities' => \$authorities,
79
    'authtypes:s' => \$authtypes,
102
    'authtypes:s' => \$authtypes,
80
    'filter=s@'     => \$filters,
103
    'filter=s@' => \$filters,
81
    'insert'        => \$insert,
104
    'insert' => \$insert,
82
    'update'        => \$update,
105
    'update' => \$update,
83
    'all'           => \$all,
106
    'all' => \$all,
84
    'match=s@'    => \$match,
107
    'match=s@' => \$match,
85
    'i|isbn' => \$isbn_check,
108
    'i|isbn' => \$isbn_check,
86
    'x:s' => \$sourcetag,
109
    'x:s' => \$sourcetag,
87
    'y:s' => \$sourcesubfield,
110
    'y:s' => \$sourcesubfield,
88
    'idmap:s' => \$idmapfl,
111
    'idmap:s' => \$idmapfl,
89
    'cleanisbn!'     => \$cleanisbn,
112
    'cleanisbn!' => \$cleanisbn,
90
    'yaml:s'        => \$yamlfile,
113
    'yaml:s' => \$yamlfile,
91
    'dedupbarcode' => \$dedup_barcode,
114
    'dedupbarcode' => \$dedup_barcode,
92
    'framework=s' => \$framework,
115
    'framework=s' => \$framework,
93
    'custom:s'    => \$localcust,
116
    'custom:s' => \$localcust,
94
    'marcmodtemplate:s' => \$marc_mod_template,
117
    'marcmodtemplate:s' => \$marc_mod_template,
95
);
118
);
119
96
$biblios ||= !$authorities;
120
$biblios ||= !$authorities;
97
$insert  ||= !$update;
121
$insert  ||= !$update;
98
my $writemode = ($append) ? "a" : "w";
122
my $writemode = ($append) ? "a" : "w";
Lines 104-109 if ($all) { Link Here
104
    $update = 1;
128
    $update = 1;
105
}
129
}
106
130
131
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
132
my $modify_biblio_marc_options = {
133
    defer_search_engine_indexing => $using_elastic_search,
134
    overlay_context => { source => 'bulkmarcimport' }
135
};
136
137
my @search_engine_record_ids;
138
my @search_engine_records;
139
my $indexer;
140
if ($using_elastic_search) {
141
    use Koha::SearchEngine::Elasticsearch::Indexer;
142
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
143
        { index => $authorities ?
144
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
145
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
146
        }
147
    );
148
}
149
107
if ($version || ($input_marc_file eq '')) {
150
if ($version || ($input_marc_file eq '')) {
108
    pod2usage( -verbose => 2 );
151
    pod2usage( -verbose => 2 );
109
    exit;
152
    exit;
Lines 112-124 if( $update && !( $match || $isbn_check ) ) { Link Here
112
    warn "Using -update without -match or -isbn seems to be useless.\n";
155
    warn "Using -update without -match or -isbn seems to be useless.\n";
113
}
156
}
114
157
115
if(defined $localcust) { #local customize module
158
if (defined $localcust) { #local customize module
116
    if(!-e $localcust) {
159
    if (!-e $localcust) {
117
        $localcust= $localcust||'LocalChanges'; #default name
160
        $localcust = $localcust || 'LocalChanges'; #default name
118
        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
161
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
119
        $localcust=~ s/\.pm$//;           #remove extension
162
        $localcust =~ s/\.pm$//;           #remove extension
120
        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
163
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
121
        if(-e $fqcust) {
164
        if (-e $fqcust) {
122
            $localcust= $fqcust;
165
            $localcust= $fqcust;
123
        }
166
        }
124
        else {
167
        else {
Lines 127-133 if(defined $localcust) { #local customize module Link Here
127
        }
170
        }
128
    }
171
    }
129
    require $localcust if $localcust;
172
    require $localcust if $localcust;
130
    $localcust=\&customize if $localcust;
173
    $localcust = \&customize if $localcust;
131
}
174
}
132
175
133
if($marc_mod_template ne '') {
176
if($marc_mod_template ne '') {
Lines 152-219 if($marc_mod_template ne '') { Link Here
152
}
195
}
153
196
154
my $dbh = C4::Context->dbh;
197
my $dbh = C4::Context->dbh;
155
my $heading_fields=get_heading_fields();
198
my $heading_fields = get_heading_fields();
156
157
my $idmapfh;
199
my $idmapfh;
200
158
if (defined $idmapfl) {
201
if (defined $idmapfl) {
159
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
202
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
160
}
203
}
161
204
162
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
205
if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
163
  $sourcetag="910";
206
    $sourcetag = "910";
164
  $sourcesubfield="a";
207
    $sourcesubfield = "a";
165
}
208
}
166
209
167
168
# Disable logging for the biblios and authorities import operation. It would unnecessarily
210
# Disable logging for the biblios and authorities import operation. It would unnecessarily
169
# slow the import
211
# slow the import
170
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
212
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
171
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
213
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
172
214
173
if ($fk_off) {
215
if ($fk_off) {
174
	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
216
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
175
}
217
}
176
218
177
178
if ($delete) {
219
if ($delete) {
179
	if ($biblios){
220
    if ($biblios) {
180
    	print "deleting biblios\n";
221
        print "Deleting biblios\n";
181
        $dbh->do("DELETE FROM biblio");
222
        $dbh->do("DELETE FROM biblio");
182
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
223
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
183
        $dbh->do("DELETE FROM biblioitems");
224
        $dbh->do("DELETE FROM biblioitems");
184
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
225
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
185
        $dbh->do("DELETE FROM items");
226
        $dbh->do("DELETE FROM items");
186
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
227
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
187
	}
228
    }
188
	else {
229
    else {
189
    	print "deleting authorities\n";
230
        print "Deleting authorities\n";
190
    	$dbh->do("truncate auth_header");
231
        $dbh->do("truncate auth_header");
191
	}
232
    }
192
    $dbh->do("truncate zebraqueue");
233
    $dbh->do("truncate zebraqueue");
193
}
234
}
194
235
195
196
197
if ($test_parameter) {
236
if ($test_parameter) {
198
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
237
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
199
}
238
}
200
239
201
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
240
my $batch;
241
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
202
242
203
# The definition of $searcher must be before MARC::Batch->new
243
# The definition of $searcher must be before MARC::Batch->new
204
my $searcher = Koha::SearchEngine::Search->new(
244
my $searcher = Koha::SearchEngine::Search->new(
205
    {
245
    {
206
        index => (
246
        index => (
207
              $authorities
247
            $authorities
208
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
248
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
209
            : $Koha::SearchEngine::BIBLIOS_INDEX
249
            : $Koha::SearchEngine::BIBLIOS_INDEX
210
        )
250
        )
211
    }
251
    }
212
);
252
);
213
253
214
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
254
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
215
my $starttime = gettimeofday;
255
my $starttime = gettimeofday;
216
my $batch;
256
217
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
257
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
218
if (defined $format && $format =~ /XML/i) {
258
if (defined $format && $format =~ /XML/i) {
219
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
259
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
Lines 226-279 if (defined $format && $format =~ /XML/i) { Link Here
226
    #       extract the records, not using regexes to look
266
    #       extract the records, not using regexes to look
227
    #       for <record>.*</record>.
267
    #       for <record>.*</record>.
228
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
268
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
229
    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
269
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
230
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
270
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
231
    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
271
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
232
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
272
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
233
    $batch = MARC::Batch->new( 'XML', $fh );
273
    $batch = MARC::Batch->new('XML', $fh);
234
} else {
274
}
235
    $batch = MARC::Batch->new( 'USMARC', $fh );
275
else {
276
    $batch = MARC::Batch->new('USMARC', $fh);
236
}
277
}
278
237
$batch->warnings_off();
279
$batch->warnings_off();
238
$batch->strict_off();
280
$batch->strict_off();
239
my $i=0;
240
my $commitnum = $commit ? $commit : 50;
281
my $commitnum = $commit ? $commit : 50;
241
my $yamlhash;
282
my $yamlhash;
242
283
243
# Skip file offset
284
# Skip file offset
244
if ( $offset ) {
285
if ($offset) {
245
    print "Skipping file offset: $offset records\n";
286
    print "Skipping file offset: $offset records\n";
246
    $batch->next() while ($offset--);
287
    $batch->next() while ($offset--);
247
}
288
}
248
289
249
my ($tagid,$subfieldid);
290
my ($tagid, $subfieldid);
250
if ($authorities){
291
if ($authorities) {
251
	  $tagid='001';
292
    $tagid = '001';
252
}
293
}
253
else {
294
else {
254
   ( $tagid, $subfieldid ) =
295
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
255
            GetMarcFromKohaField( "biblio.biblionumber" );
296
    $tagid ||= "001";
256
	$tagid||="001";
257
}
297
}
258
298
299
my $sth_isbn;
259
# the SQL query to search on isbn
300
# the SQL query to search on isbn
260
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
301
if ($isbn_check) {
302
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
303
}
261
304
262
my $loghandle;
305
my $loghandle;
263
if ($logfile){
306
if ($logfile) {
264
   $loghandle= IO::File->new($logfile, $writemode) ;
307
    $loghandle= IO::File->new($logfile, $writemode);
265
   print $loghandle "id;operation;status\n";
308
    print $loghandle "id;operation;status\n";
266
}
309
}
267
310
311
my $record_number = 0;
268
my $logger = Koha::Logger->get;
312
my $logger = Koha::Logger->get;
269
my $schema = Koha::Database->schema;
313
my $schema = Koha::Database->schema;
270
$schema->txn_begin;
314
my $marc_records = [];
271
RECORD: while (  ) {
315
RECORD: while (  ) {
272
    my $record;
316
    my $record;
273
    # get records
317
    $record_number++;
318
    # get record
274
    eval { $record = $batch->next() };
319
    eval { $record = $batch->next() };
275
    if ( $@ ) {
320
    if ($@) {
276
        print "Bad MARC record $i: $@ skipped\n";
321
        print "Bad MARC record $record_number: $@ skipped\n";
277
        # FIXME - because MARC::Batch->next() combines grabbing the next
322
        # FIXME - because MARC::Batch->next() combines grabbing the next
278
        # blob and parsing it into one operation, a correctable condition
323
        # blob and parsing it into one operation, a correctable condition
279
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
324
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
Lines 282-560 RECORD: while ( ) { Link Here
282
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
327
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
283
        next;
328
        next;
284
    }
329
    }
285
    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
330
    if ($record) {
286
    last unless ( $record );
331
        # transcode the record to UTF8 if needed & applicable.
287
    $i++;
332
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
288
    if( ($verbose//1)==1 ) { #no dot for verbose==2
333
            my ($guessed_charset, $charset_errors);
289
        print "." . ( $i % 100==0 ? "\n$i" : '' );
334
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
335
            if ($guessed_charset eq 'failed') {
336
                warn "ERROR: failed to perform character conversion for record $record_number\n";
337
                next RECORD;
338
            }
339
        }
340
        SetUTF8Flag($record);
341
        &$localcust($record) if $localcust;
342
        push @{$marc_records}, $record;
343
    }
344
    else {
345
        last;
290
    }
346
    }
347
}
291
348
292
    # transcode the record to UTF8 if needed & applicable.
349
$record_number = 0;
293
    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
350
my $records_total = @{$marc_records};
294
        # FIXME update condition
351
$schema->txn_begin;
295
        my ($guessed_charset, $charset_errors);
352
RECORD: foreach my $record (@{$marc_records}) {
296
         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
353
    $record_number++;
297
        if ($guessed_charset eq 'failed') {
354
    if (($verbose//1) == 1) { #no dot for verbose==2
298
            warn "ERROR: failed to perform character conversion for record $i\n";
355
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
299
            next RECORD;            
300
        }
301
    }
356
    }
302
    SetUTF8Flag($record);
357
303
    if($marc_mod_template_id > 0) {
358
    if ($marc_mod_template_id > 0) {
304
    print "Modifying MARC\n" if $verbose;
359
        print "Modifying MARC\n" if $verbose;
305
    ModifyRecordWithTemplate( $marc_mod_template_id, $record );
360
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
306
    }
361
    }
307
    &$localcust($record) if $localcust;
362
308
    my $isbn;
363
    my $isbn;
309
    # remove trailing - in isbn (only for biblios, of course)
364
    # remove trailing - in isbn (only for biblios, of course)
310
    if( $biblios ) {
365
    if ($biblios && ($cleanisbn || $isbn_check)) {
311
        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
366
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
312
        my $field = $record->field($tag);
367
        my $field = $record->field($tag);
313
        $isbn = $field && $field->subfield('a');
368
        $isbn = $field && $field->subfield('a');
314
        if ( $isbn && $cleanisbn ) {
369
        if ($isbn && $cleanisbn) {
315
            $isbn =~ s/-//g;
370
            $isbn =~ s/-//g;
316
            $field->update('a' => $isbn);
371
            $field->update('a' => $isbn);
317
        }
372
        }
318
    }
373
    }
319
    my $id;
320
    # search for duplicates (based on Local-number)
374
    # search for duplicates (based on Local-number)
321
    my $originalid;
375
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
322
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
376
    my $matched_record_id = undef;
323
    if ($match) {
377
    if ($match) {
324
        require C4::Search;
378
        require C4::Search;
325
        my $query = build_query( $match, $record );
379
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
326
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
380
        my $query = build_query($match, $record);
327
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
381
        $logger->debug("Bulkmarcimport: $query");
382
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
328
        # changed to warn so able to continue with one broken record
383
        # changed to warn so able to continue with one broken record
329
        if ( defined $error ) {
384
        if (defined $error) {
330
            warn "unable to search the database for duplicates : $error";
385
            warn "unable to search the database for duplicates : $error";
331
            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
386
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
332
            next RECORD;
387
            next RECORD;
333
        }
388
        }
334
        if ( $results && scalar(@$results) == 1 ) {
389
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
335
            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
390
        # sub SimpleSearch could return undefined, but only on error, so
336
            SetUTF8Flag($marcrecord);
391
        # should not really need to safeguard here, but do so anyway
337
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
392
        $results //= [];
338
            if ( $authorities && $marcFlavour ) {
393
        if (@{$results} == 1) {
339
                #Skip if authority in database is the same as the on in database
394
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
340
                if ( $marcrecord->field('005') && $record->field('005') &&
395
            SetUTF8Flag($matched_record);
341
                     $marcrecord->field('005')->data && $record->field('005')->data &&
396
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
342
                     $marcrecord->field('005')->data >= $record->field('005')->data ) {
397
398
            if ($authorities && $marc_flavour) {
399
                #Skip if authority in database is the same or newer than the incoming record
400
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
343
                    if ($yamlfile) {
401
                    if ($yamlfile) {
344
                        $yamlhash->{$originalid}->{'authid'} = $id;
402
                        $yamlhash->{$originalid} = YAMLFileEntry(
345
403
                            $matched_record,
346
                        # we recover all subfields of the heading authorities
404
                            $matched_record_id,
347
                        my @subfields;
405
                            0
348
                        foreach my $field ( $marcrecord->field("2..") ) {
406
                        );
349
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
350
                        }
351
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
352
                        $yamlhash->{$originalid}->{'updated'} = 0;
353
                    }
407
                    }
354
                    next;
408
                    next;
355
                }
409
                }
356
            }
410
            }
357
        } elsif ( $results && scalar(@$results) > 1 ) {
358
            $logger->debug("more than one match for $query");
359
        } else {
360
            $logger->debug("nomatch for $query");
361
        }
411
        }
362
    }
412
        elsif(@{$results} > 1) {
363
    if ($keepids && $originalid) {
413
            $logger->debug("More than one match for: $query");
414
        }
415
        else {
416
            $logger->debug("No match for: $query");
417
        }
418
419
        if ($keepids && $originalid) {
364
            my $storeidfield;
420
            my $storeidfield;
365
            if ( length($keepids) == 3 ) {
421
            if (length($keepids) == 3) {
366
                $storeidfield = MARC::Field->new( $keepids, $originalid );
422
                $storeidfield = MARC::Field->new($keepids, $originalid);
367
            } else {
423
            } else {
368
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
424
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
369
            }
425
            }
370
            $record->insert_fields_ordered($storeidfield);
426
            $record->insert_fields_ordered($storeidfield);
371
            $record->delete_field( $record->field($tagid) );
427
            $record->delete_field($record->field($tagid));
428
        }
372
    }
429
    }
430
373
    foreach my $stringfilter (@$filters) {
431
    foreach my $stringfilter (@$filters) {
374
        if ( length($stringfilter) == 3 ) {
432
        if (length($stringfilter) == 3) {
375
            foreach my $field ( $record->field($stringfilter) ) {
433
            foreach my $field ($record->field($stringfilter)) {
376
                $record->delete_field($field);
434
                $record->delete_field($field);
377
                $logger->debug("removed : ", $field->as_string);
435
                $logger->debug("Removed: ", $field->as_string);
378
            }
436
            }
379
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
437
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
380
            my $removetag = $1;
438
            my $removetag = $1;
381
            my $removesubfield = $2;
439
            my $removesubfield = $2;
382
            my $removematch = $3;
440
            my $removematch = $3;
383
            if ( ( $removetag > "010" ) && $removesubfield ) {
441
            if (($removetag > "010") && $removesubfield) {
384
                foreach my $field ( $record->field($removetag) ) {
442
                foreach my $field ($record->field($removetag)) {
385
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
443
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
386
                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));
444
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
387
                }
445
                }
388
            }
446
            }
389
        }
447
        }
390
    }
448
    }
391
    unless ($test_parameter) {
449
    unless ($test_parameter) {
392
        if ($authorities){
450
        if ($authorities) {
393
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
451
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
394
            my $authid= ($id?$id:GuessAuthId($record));
452
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
395
            if ($authid && GetAuthority($authid) && $update ){
453
            if ($authid && GetAuthority($authid) && $update) {
396
            ## Authority has an id and is in database : Replace
454
            ## Authority has an id and is in database : Replace
397
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
455
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
398
                if ($@){
456
                if ($@) {
399
                    warn "Problem with authority $authid Cannot Modify";
457
                    warn "Problem with authority $authid Cannot Modify";
400
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
458
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
401
                }
459
                }
402
				else{
460
                else{
403
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
461
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
404
				}
462
                }
405
            }  
463
            }
406
	        else {
464
            else {
407
            ## True insert in database
465
            ## True insert in database
408
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
466
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
409
                if ($@){
467
                if ($@) {
410
                    warn "Problem with authority $authid Cannot Add".$@;
468
                    warn "Problem with authority $originalid Cannot Add ".$@;
411
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
469
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
412
                }
470
                }
413
   				else{
471
                else {
414
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
472
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
415
				}
473
                }
416
 	        }
474
417
            if ($yamlfile) {
418
            $yamlhash->{$originalid}->{'authid'} = $authid;
419
            my @subfields;
420
            foreach my $field ( $record->field("2..") ) {
421
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
422
            }
475
            }
423
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
476
            if ($yamlfile) {
424
            $yamlhash->{$originalid}->{'updated'} = 1;
477
                $yamlhash->{$originalid} = YAMLFileEntry(
478
                    $record,
479
                    $authid,
480
                    1 #@FIXME: Really always updated?
481
                );
425
            }
482
            }
426
        }
483
        }
427
        else {
484
        else {
428
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
485
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
429
            $biblionumber = $id;
430
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
486
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
431
            if (!$biblionumber && $isbn_check && $isbn) {
487
            if (!$matched_record_id && $isbn_check && $isbn) {
432
    #         warn "search ISBN : $isbn";
433
                $sth_isbn->execute($isbn);
488
                $sth_isbn->execute($isbn);
434
                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
489
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
490
            }
491
492
            if (defined $idmapfl && $matched_record_id) {
493
                if ($sourcetag < "010") {
494
                    if ($record->field($sourcetag)) {
495
                        my $source = $record->field($sourcetag)->data();
496
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
497
                    }
498
                }
499
                else {
500
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
501
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
502
                }
435
            }
503
            }
436
        	if (defined $idmapfl) {
504
437
			 	if ($sourcetag < "010"){
505
            # create biblio, unless we already have it ( either match or isbn )
438
					if ($record->field($sourcetag)){
506
            if ($matched_record_id) {
439
					  my $source = $record->field($sourcetag)->data();
507
                # TODO: Implement also for authority records!
440
                      printf($idmapfh "%s|%s\n",$source,$biblionumber);
441
					}
442
			    } else {
443
					my $source=$record->subfield($sourcetag,$sourcesubfield);
444
                    printf($idmapfh "%s|%s\n",$source,$biblionumber);
445
			  }
446
			}
447
					# create biblio, unless we already have it ( either match or isbn )
448
            if ($biblionumber) {
449
                eval{
508
                eval{
450
                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
509
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
451
                };
510
                };
452
                if ($update) {
511
                if ($update) {
453
                    eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) };
512
                    my $success;
513
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
454
                    if ($@) {
514
                    if ($@) {
455
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
515
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
456
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
516
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
457
                        next RECORD;
517
                        next RECORD;
458
                    } else {
459
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
460
                    }
518
                    }
461
                } else {
519
                    elsif (!$success) {
462
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
520
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
463
                }
521
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
464
            } else {
465
                if ($insert) {
466
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
467
                    if ($@) {
468
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
469
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
470
                        next RECORD;
522
                        next RECORD;
471
                    } else {
472
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
473
                    }
523
                    }
474
                } else {
524
                    else {
475
                    warn "WARNING: Updating record ".($id||$originalid)." failed";
525
                        $record_id = $matched_record_id;
476
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
526
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
527
                    }
528
                }
529
                else {
530
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
531
                }
532
            }
533
            elsif ($insert) {
534
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
535
                if ($@) {
536
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
537
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
477
                    next RECORD;
538
                    next RECORD;
478
                }
539
                }
540
                else {
541
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
542
                }
479
            }
543
            }
480
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
544
            else {
481
            my $error_adding = $@;
545
                warn "WARNING: Updating record ".($originalid)." failed";
482
            # Work on a clone so that if there are real errors, we can maybe
546
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
483
            # fix them up later.
484
			my $clone_record = $record->clone();
485
            C4::Biblio::_strip_item_fields($clone_record, '');
486
            # This sets the marc fields if there was an error, and also calls
487
            # defer_marc_save.
488
            ModBiblioMarc( $clone_record, $biblionumber );
489
            if ( $error_adding ) {
490
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
491
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
492
                # if we failed because of an exception, assume that 
493
                # the MARC columns in biblioitems were not set.
494
                next RECORD;
547
                next RECORD;
495
            }
548
            }
496
 			else{
549
            my $record_has_added_items = 0;
497
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
550
            if ($record_id) {
498
			}
551
                $yamlhash->{$originalid} = $record_id if $yamlfile;
499
            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
552
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
500
                # Find the record called 'barcode'
553
                $record_has_added_items = @{$itemnumbers_ref};
501
                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
554
                my $error_adding = $@;
502
                # Now remove any items that didn't have a duplicate_barcode error,
555
                # Work on a clone so that if there are real errors, we can maybe
503
                # erase the barcodes on items that did, and re-add those items.
556
                # fix them up later.
504
                my %dupes;
557
                my $clone_record = $record->clone();
505
                foreach my $i (0 .. $#{$errors_ref}) {
558
                C4::Biblio::_strip_item_fields($clone_record, $framework);
506
                    my $ref = $errors_ref->[$i];
559
                # This sets the marc fields if there was an error, and also calls
507
                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
560
                # defer_marc_save.
508
                        $dupes{$ref->{item_sequence}} = 1;
561
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
509
                        # Delete the error message because we're going to
562
                if ($error_adding) {
510
                        # retry this one.
563
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
511
                        delete $errors_ref->[$i];
564
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
512
                    }
513
                }
514
                my $seq = 0;
515
                foreach my $field ($record->field($tag)) {
516
                    $seq++;
517
                    if ($dupes{$seq}) {
518
                        # Here we remove the barcode
519
                        $field->delete_subfield(code => $sub);
520
                    } else {
521
                        # otherwise we delete the field because we don't want
522
                        # two of them
523
                        $record->delete_fields($field);
524
                    }
525
                }
526
                # Now re-add the record as before, adding errors to the prev list
527
                my $more_errors;
528
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
529
                if ( $@ ) {
530
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
531
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
532
                    # if we failed because of an exception, assume that
565
                    # if we failed because of an exception, assume that
533
                    # the MARC columns in biblioitems were not set.
566
                    # the MARC columns in biblioitems were not set.
534
                    ModBiblioMarc( $record, $biblionumber );
535
                    next RECORD;
567
                    next RECORD;
536
                } else {
537
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
538
                }
568
                }
539
                push @$errors_ref, @{ $more_errors };
569
                else {
540
            }
570
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
541
            if ($#{ $errors_ref } > -1) {
571
                }
542
                report_item_errors($biblionumber, $errors_ref);
572
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
573
                    # Find the record called 'barcode'
574
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
575
                    # Now remove any items that didn't have a duplicate_barcode error,
576
                    # erase the barcodes on items that did, and re-add those items.
577
                    my %dupes;
578
                    foreach my $i (0 .. $#{$errors_ref}) {
579
                        my $ref = $errors_ref->[$i];
580
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
581
                            $dupes{$ref->{item_sequence}} = 1;
582
                            # Delete the error message because we're going to
583
                            # retry this one.
584
                            delete $errors_ref->[$i];
585
                        }
586
                    }
587
                    my $seq = 0;
588
                    foreach my $field ($record->field($tag)) {
589
                        $seq++;
590
                        if ($dupes{$seq}) {
591
                            # Here we remove the barcode
592
                            $field->delete_subfield(code => $sub);
593
                        }
594
                        else {
595
                            # otherwise we delete the field because we don't want
596
                            # two of them
597
                            $record->delete_fields($field);
598
                        }
599
                    }
600
                    # Now re-add the record as before, adding errors to the prev list
601
                    my $more_errors;
602
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
603
                    $record_has_added_items ||= @{$itemnumbers_ref};
604
                    if ($@) {
605
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
606
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
607
                        # if we failed because of an exception, assume that
608
                        # the MARC columns in biblioitems were not set.
609
610
                        # @FIXME: Why do we save here without stripping items? Besides,
611
                        # save with stripped items has already been performed
612
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
613
                        next RECORD;
614
                    }
615
                    else {
616
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
617
                    }
618
                    push @$errors_ref, @{$more_errors};
619
                }
620
                if (@{$errors_ref}) {
621
                    report_item_errors($record_id, $errors_ref);
622
                }
623
                C4::Biblio::_strip_item_fields($record, $framework);
624
                if ($record_has_added_items || $matched_record_id) {
625
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
626
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
627
                }
628
                push @search_engine_record_ids, $record_id;
629
                push @search_engine_records, $record;
543
            }
630
            }
544
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
545
        }
631
        }
546
        if ( 0 == $i % $commitnum ) {
632
        if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) {
547
            $schema->txn_commit;
633
            $schema->txn_commit;
548
            $schema->txn_begin;
634
            $schema->txn_begin;
635
            if ($indexer) {
636
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
637
                @search_engine_record_ids = ();
638
                @search_engine_records = ();
639
            }
549
        }
640
        }
550
    }
641
    }
551
    print $record->as_formatted()."\n" if ($verbose//0)==2;
642
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
552
    last if $i == $number;
643
    last if $record_number == $number;
553
}
644
}
554
$schema->txn_commit;
645
$schema->txn_commit;
555
646
556
if ($fk_off) {
647
if ($fk_off) {
557
	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
648
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
558
}
649
}
559
650
560
# Restore CataloguingLog and AuthoritiesLog
651
# Restore CataloguingLog and AuthoritiesLog
Lines 562-572 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; Link Here
562
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
653
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
563
654
564
my $timeneeded = gettimeofday - $starttime;
655
my $timeneeded = gettimeofday - $starttime;
565
print "\n$i MARC records done in $timeneeded seconds\n";
656
print "\n$record_number MARC records done in $timeneeded seconds\n";
566
if ($logfile){
657
if ($logfile) {
567
  print $loghandle "file : $input_marc_file\n";
658
    print $loghandle "file : $input_marc_file\n";
568
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
659
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
569
  $loghandle->close;
660
    $loghandle->close;
570
}
661
}
571
if ($yamlfile) {
662
if ($yamlfile) {
572
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
663
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
Lines 574-620 if ($yamlfile) { Link Here
574
}
665
}
575
exit 0;
666
exit 0;
576
667
577
sub GetRecordId{
668
sub YAMLFileEntry {
578
	my $marcrecord=shift;
669
    my ($record, $record_id, $updated) = @_;
579
	my $tag=shift;
670
580
	my $subfield=shift;
671
    my $entry = {
581
	my $id;
672
        authid => $record_id
582
	if ($tag lt "010"){
673
    };
583
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
674
584
	} 
675
    # we recover all subfields of the heading authorities
585
	elsif ($subfield){
676
    my @subfields;
586
		if ($marcrecord->field($tag)){
677
    foreach my $field ($record->field("2..")) {
587
			return $marcrecord->subfield($tag,$subfield);
678
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
588
		}
679
    }
589
	}
680
    $entry->{'subfields'} = \@subfields;
590
	return $id;
681
    $entry->{'updated'} = $updated;
682
683
    return $entry;
591
}
684
}
592
sub build_query {
685
593
	my $match = shift;
686
sub RecordRevisionIsGtOrEq {
594
	my $record=shift;
687
    my ($record_a, $record_b) = @_;
595
        my @searchstrings;
688
    return $record_a->field('005') && $record_b->field('005') &&
596
	foreach my $matchingpoint (@$match){
689
    $record_a->field('005')->data && $record_b->field('005')->data &&
597
	  my $string = build_simplequery($matchingpoint,$record);
690
    $record_a->field('005')->data >= $record_b->field('005')->data;
598
	  push @searchstrings,$string if (length($string)>0);
691
}
692
693
sub GetRecordId {
694
    my $marcrecord = shift;
695
    my $tag = shift;
696
    my $subfield = shift;
697
    if ($tag lt "010") {
698
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
699
    }
700
    elsif ($subfield) {
701
        if ($marcrecord->field($tag)) {
702
            return $marcrecord->subfield($tag, $subfield);
599
        }
703
        }
704
    }
705
    return undef;
706
}
707
sub build_query {
708
    my ($match, $record) = @_;
709
    my @searchstrings;
710
711
    foreach my $matchpoint (@$match) {
712
        my $query = build_simplequery($matchpoint, $record);
713
        push (@searchstrings, $query) if $query;
714
    }
600
    my $op = 'and';
715
    my $op = 'and';
601
    return join(" $op ",@searchstrings);
716
    return join(" $op ", @searchstrings);
602
}
717
}
603
sub build_simplequery {
718
sub build_simplequery {
604
	my $element=shift;
719
    my ($matchpoint, $record) = @_;
605
	my $record=shift;
720
606
    my @searchstrings;
721
    my @searchstrings;
607
    my ($index,$recorddata)=split /,/,$element;
722
    my ($index, $record_data) = split (/,/, $matchpoint);
608
    if ($recorddata=~/(\d{3})(.*)/) {
723
    if ($record_data =~ /(\d{3})(.*)/) {
609
        my ($tag,$subfields) =($1,$2);
724
        my ($tag, $subfields) = ($1, $2);
610
        foreach my $field ($record->field($tag)){
725
        foreach my $field ($record->field($tag)) {
611
		  if (length($field->as_string("$subfields"))>0){
726
            if (length($field->as_string("$subfields")) > 0) {
612
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
727
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
613
		  }
728
            }
614
        }
729
        }
615
    }
730
    }
731
    else {
732
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
733
    }
616
    my $op = 'and';
734
    my $op = 'and';
617
    return join(" $op ",@searchstrings);
735
    return join(" $op ", @searchstrings);
618
}
736
}
619
sub report_item_errors {
737
sub report_item_errors {
620
    my $biblionumber = shift;
738
    my $biblionumber = shift;
Lines 629-648 sub report_item_errors { Link Here
629
        print $msg, "\n";
747
        print $msg, "\n";
630
    }
748
    }
631
}
749
}
632
sub printlog{
750
sub printlog {
633
	my $logelements=shift;
751
    my $logelements = shift;
634
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
752
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
635
}
753
}
636
sub get_heading_fields{
754
sub get_heading_fields {
637
    my $headingfields;
755
    my $headingfields;
638
    if ($authtypes){
756
    if ($authtypes) {
639
        $headingfields = YAML::XS::LoadFile($authtypes);
757
        $headingfields = YAML::XS::LoadFile($authtypes);
640
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
758
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
641
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
759
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
642
    }
760
    }
643
    unless ($headingfields){
761
    unless ($headingfields) {
644
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
762
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
645
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
763
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
646
    }
764
    }
647
    return $headingfields;
765
    return $headingfields;
648
}
766
}
649
- 

Return to bug 29440