View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

(-)a/C4/Biblio.pm (-5 / +28 lines)
Lines 343-348 Unless C<disable_autolink> is passed ModBiblio will relink record headings Link Here
343
to authorities based on settings in the system preferences. This flag allows
343
to authorities based on settings in the system preferences. This flag allows
344
us to not relink records when the authority linker is saving modifications.
344
us to not relink records when the authority linker is saving modifications.
345
345
346
=item C<defer_search_engine_indexing>
347
348
Don't update search index. Useful for bulk updates where this is handled
349
manually for optimization purposes.
350
346
=back
351
=back
347
352
348
Returns 1 on success 0 on failure
353
Returns 1 on success 0 on failure
Lines 352-357 Returns 1 on success 0 on failure Link Here
352
sub ModBiblio {
357
sub ModBiblio {
353
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
358
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
354
    $options //= {};
359
    $options //= {};
360
    my %mod_biblio_marc_options;
361
    $mod_biblio_marc_options{'defer_search_engine_indexing'} =
362
        exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
355
363
356
    if (!$record) {
364
    if (!$record) {
357
        carp 'No record passed to ModBiblio';
365
        carp 'No record passed to ModBiblio';
Lines 415-421 sub ModBiblio { Link Here
415
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
423
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
416
424
417
    # update the MARC record (that now contains biblio and items) with the new record data
425
    # update the MARC record (that now contains biblio and items) with the new record data
418
    ModBiblioMarc( $record, $biblionumber );
426
    ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
419
427
420
    # modify the other koha tables
428
    # modify the other koha tables
421
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
429
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2849-2866 sub _koha_delete_biblio_metadata { Link Here
2849
2857
2850
=head2 ModBiblioMarc
2858
=head2 ModBiblioMarc
2851
2859
2852
  ModBiblioMarc($newrec,$biblionumber);
2860
  ModBiblioMarc($newrec, $biblionumber, $options);
2853
2861
2854
Add MARC XML data for a biblio to koha
2862
Add MARC XML data for a biblio to koha
2855
2863
2856
Function exported, but should NOT be used, unless you really know what you're doing
2864
Function exported, but should NOT be used, unless you really know what you're doing
2857
2865
2866
The C<$options> argument is a hashref with additional parameters:
2867
2868
=over 4
2869
2870
=item C<defer_search_engine_indexing>
2871
2872
Don't update search index. Useful for bulk updates where this is handled
2873
manually for optimization purposes.
2874
2875
=back
2876
2858
=cut
2877
=cut
2859
2878
2860
sub ModBiblioMarc {
2879
sub ModBiblioMarc {
2861
    # pass the MARC::Record to this function, and it will create the records in
2880
    # pass the MARC::Record to this function, and it will create the records in
2862
    # the marcxml field
2881
    # the marcxml field
2863
    my ( $record, $biblionumber ) = @_;
2882
    my ( $record, $biblionumber, $options ) = @_;
2883
    $options //= {};
2884
2864
    if ( !$record ) {
2885
    if ( !$record ) {
2865
        carp 'ModBiblioMarc passed an undefined record';
2886
        carp 'ModBiblioMarc passed an undefined record';
2866
        return;
2887
        return;
Lines 2925-2932 sub ModBiblioMarc { Link Here
2925
    $m_rs->metadata( $record->as_xml_record($encoding) );
2946
    $m_rs->metadata( $record->as_xml_record($encoding) );
2926
    $m_rs->store;
2947
    $m_rs->store;
2927
2948
2928
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
2949
    unless (exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'}) {
2929
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
2950
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
2951
        $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
2952
    }
2930
2953
2931
    return $biblionumber;
2954
    return $biblionumber;
2932
}
2955
}
(-)a/misc/migration_tools/bulkmarcimport.pl (-308 / +425 lines)
Lines 17-22 use C4::Biblio qw( Link Here
17
    GetMarcFromKohaField
17
    GetMarcFromKohaField
18
    ModBiblio
18
    ModBiblio
19
    ModBiblioMarc
19
    ModBiblioMarc
20
    GetFrameworkCode
21
    GetMarcBiblio
20
);
22
);
21
use C4::Koha;
23
use C4::Koha;
22
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
24
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
Lines 40-61 use Koha::SearchEngine; Link Here
40
use Koha::SearchEngine::Search;
42
use Koha::SearchEngine::Search;
41
43
42
use open qw( :std :encoding(UTF-8) );
44
use open qw( :std :encoding(UTF-8) );
43
binmode( STDOUT, ":encoding(UTF-8)" );
45
binmode(STDOUT, ":encoding(UTF-8)");
44
my ( $input_marc_file, $number, $offset) = ('',0,0);
46
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
45
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
47
my $version;
46
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
48
my $delete;
47
my $cleanisbn = 1;
49
my $test_parameter;
48
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
50
my $skip_marc8_conversion;
51
my $char_encoding;
52
my $verbose;
53
my $commit;
54
my $fk_off;
55
my $format;
56
my $biblios;
57
my $authorities;
58
my $keepids;
59
my $match;
60
my $isbn_check;
61
my $logfile;
62
my $insert;
63
my $filters;
64
my $update;
65
my $all;
66
my $yamlfile;
67
my $authtypes;
68
my $append;
69
my $sourcetag;
70
my $sourcesubfield;
71
my $idmapfl;
72
my $dedup_barcode;
49
my $framework = '';
73
my $framework = '';
50
my $localcust;
74
my $localcust;
51
my $marc_mod_template = '';
75
my $marc_mod_template = '';
52
my $marc_mod_template_id = -1;
76
my $marc_mod_template_id = -1;
53
77
$| = 1;
54
$|=1;
55
78
56
GetOptions(
79
GetOptions(
57
    'commit:f'    => \$commit,
80
    'commit:f' => \$commit,
58
    'file:s'    => \$input_marc_file,
81
    'file:s' => \$input_marc_file,
59
    'n:f' => \$number,
82
    'n:f' => \$number,
60
    'o|offset:f' => \$offset,
83
    'o|offset:f' => \$offset,
61
    'h' => \$version,
84
    'h' => \$version,
Lines 72-93 GetOptions( Link Here
72
    'b|biblios' => \$biblios,
95
    'b|biblios' => \$biblios,
73
    'a|authorities' => \$authorities,
96
    'a|authorities' => \$authorities,
74
    'authtypes:s' => \$authtypes,
97
    'authtypes:s' => \$authtypes,
75
    'filter=s@'     => \$filters,
98
    'filter=s@' => \$filters,
76
    'insert'        => \$insert,
99
    'insert' => \$insert,
77
    'update'        => \$update,
100
    'update' => \$update,
78
    'all'           => \$all,
101
    'all' => \$all,
79
    'match=s@'    => \$match,
102
    'match=s@' => \$match,
80
    'i|isbn' => \$isbn_check,
103
    'i|isbn' => \$isbn_check,
81
    'x:s' => \$sourcetag,
104
    'x:s' => \$sourcetag,
82
    'y:s' => \$sourcesubfield,
105
    'y:s' => \$sourcesubfield,
83
    'idmap:s' => \$idmapfl,
106
    'idmap:s' => \$idmapfl,
84
    'cleanisbn!'     => \$cleanisbn,
107
    'cleanisbn!' => \$cleanisbn,
85
    'yaml:s'        => \$yamlfile,
108
    'yaml:s' => \$yamlfile,
86
    'dedupbarcode' => \$dedup_barcode,
109
    'dedupbarcode' => \$dedup_barcode,
87
    'framework=s' => \$framework,
110
    'framework=s' => \$framework,
88
    'custom:s'    => \$localcust,
111
    'custom:s' => \$localcust,
89
    'marcmodtemplate:s' => \$marc_mod_template,
112
    'marcmodtemplate:s' => \$marc_mod_template,
90
);
113
);
114
91
$biblios ||= !$authorities;
115
$biblios ||= !$authorities;
92
$insert  ||= !$update;
116
$insert  ||= !$update;
93
my $writemode = ($append) ? "a" : "w";
117
my $writemode = ($append) ? "a" : "w";
Lines 99-104 if ($all) { Link Here
99
    $update = 1;
123
    $update = 1;
100
}
124
}
101
125
126
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
127
my $modify_biblio_marc_options = {
128
    defer_search_engine_indexing => $using_elastic_search,
129
    overlay_context => { source => 'bulkmarcimport' }
130
};
131
132
my @search_engine_record_ids;
133
my @search_engine_records;
134
my $indexer;
135
if ($using_elastic_search) {
136
    use Koha::SearchEngine::Elasticsearch::Indexer;
137
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
138
        { index => $authorities ?
139
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
140
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
141
        }
142
    );
143
}
144
102
if ($version || ($input_marc_file eq '')) {
145
if ($version || ($input_marc_file eq '')) {
103
    pod2usage( -verbose => 2 );
146
    pod2usage( -verbose => 2 );
104
    exit;
147
    exit;
Lines 107-119 if( $update && !( $match || $isbn_check ) ) { Link Here
107
    warn "Using -update without -match or -isbn seems to be useless.\n";
150
    warn "Using -update without -match or -isbn seems to be useless.\n";
108
}
151
}
109
152
110
if(defined $localcust) { #local customize module
153
if (defined $localcust) { #local customize module
111
    if(!-e $localcust) {
154
    if (!-e $localcust) {
112
        $localcust= $localcust||'LocalChanges'; #default name
155
        $localcust = $localcust || 'LocalChanges'; #default name
113
        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
156
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
114
        $localcust=~ s/\.pm$//;           #remove extension
157
        $localcust =~ s/\.pm$//;           #remove extension
115
        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
158
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
116
        if(-e $fqcust) {
159
        if (-e $fqcust) {
117
            $localcust= $fqcust;
160
            $localcust= $fqcust;
118
        }
161
        }
119
        else {
162
        else {
Lines 122-128 if(defined $localcust) { #local customize module Link Here
122
        }
165
        }
123
    }
166
    }
124
    require $localcust if $localcust;
167
    require $localcust if $localcust;
125
    $localcust=\&customize if $localcust;
168
    $localcust = \&customize if $localcust;
126
}
169
}
127
170
128
if($marc_mod_template ne '') {
171
if($marc_mod_template ne '') {
Lines 147-214 if($marc_mod_template ne '') { Link Here
147
}
190
}
148
191
149
my $dbh = C4::Context->dbh;
192
my $dbh = C4::Context->dbh;
150
my $heading_fields=get_heading_fields();
193
my $heading_fields = get_heading_fields();
151
152
my $idmapfh;
194
my $idmapfh;
195
153
if (defined $idmapfl) {
196
if (defined $idmapfl) {
154
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
197
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
155
}
198
}
156
199
157
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
200
if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
158
  $sourcetag="910";
201
    $sourcetag = "910";
159
  $sourcesubfield="a";
202
    $sourcesubfield = "a";
160
}
203
}
161
204
162
163
# Disable logging for the biblios and authorities import operation. It would unnecessarily
205
# Disable logging for the biblios and authorities import operation. It would unnecessarily
164
# slow the import
206
# slow the import
165
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
207
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
166
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
208
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
167
209
168
if ($fk_off) {
210
if ($fk_off) {
169
	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
211
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
170
}
212
}
171
213
172
173
if ($delete) {
214
if ($delete) {
174
	if ($biblios){
215
    if ($biblios) {
175
    	print "deleting biblios\n";
216
        print "Deleting biblios\n";
176
        $dbh->do("DELETE FROM biblio");
217
        $dbh->do("DELETE FROM biblio");
177
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
218
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
178
        $dbh->do("DELETE FROM biblioitems");
219
        $dbh->do("DELETE FROM biblioitems");
179
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
220
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
180
        $dbh->do("DELETE FROM items");
221
        $dbh->do("DELETE FROM items");
181
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
222
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
182
	}
223
    }
183
	else {
224
    else {
184
    	print "deleting authorities\n";
225
        print "Deleting authorities\n";
185
    	$dbh->do("truncate auth_header");
226
        $dbh->do("truncate auth_header");
186
	}
227
    }
187
    $dbh->do("truncate zebraqueue");
228
    $dbh->do("truncate zebraqueue");
188
}
229
}
189
230
190
191
192
if ($test_parameter) {
231
if ($test_parameter) {
193
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
232
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
194
}
233
}
195
234
196
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
235
my $batch;
236
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
197
237
198
# The definition of $searcher must be before MARC::Batch->new
238
# The definition of $searcher must be before MARC::Batch->new
199
my $searcher = Koha::SearchEngine::Search->new(
239
my $searcher = Koha::SearchEngine::Search->new(
200
    {
240
    {
201
        index => (
241
        index => (
202
              $authorities
242
            $authorities
203
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
243
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
204
            : $Koha::SearchEngine::BIBLIOS_INDEX
244
            : $Koha::SearchEngine::BIBLIOS_INDEX
205
        )
245
        )
206
    }
246
    }
207
);
247
);
208
248
209
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
249
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
210
my $starttime = gettimeofday;
250
my $starttime = gettimeofday;
211
my $batch;
251
212
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
252
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
213
if (defined $format && $format =~ /XML/i) {
253
if (defined $format && $format =~ /XML/i) {
214
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
254
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
Lines 221-274 if (defined $format && $format =~ /XML/i) { Link Here
221
    #       extract the records, not using regexes to look
261
    #       extract the records, not using regexes to look
222
    #       for <record>.*</record>.
262
    #       for <record>.*</record>.
223
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
263
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
224
    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
264
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
225
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
265
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
226
    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
266
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
227
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
267
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
228
    $batch = MARC::Batch->new( 'XML', $fh );
268
    $batch = MARC::Batch->new('XML', $fh);
229
} else {
269
}
230
    $batch = MARC::Batch->new( 'USMARC', $fh );
270
else {
271
    $batch = MARC::Batch->new('USMARC', $fh);
231
}
272
}
273
232
$batch->warnings_off();
274
$batch->warnings_off();
233
$batch->strict_off();
275
$batch->strict_off();
234
my $i=0;
235
my $commitnum = $commit ? $commit : 50;
276
my $commitnum = $commit ? $commit : 50;
236
my $yamlhash;
277
my $yamlhash;
237
278
238
# Skip file offset
279
# Skip file offset
239
if ( $offset ) {
280
if ($offset) {
240
    print "Skipping file offset: $offset records\n";
281
    print "Skipping file offset: $offset records\n";
241
    $batch->next() while ($offset--);
282
    $batch->next() while ($offset--);
242
}
283
}
243
284
244
my ($tagid,$subfieldid);
285
my ($tagid, $subfieldid);
245
if ($authorities){
286
if ($authorities) {
246
	  $tagid='001';
287
    $tagid = '001';
247
}
288
}
248
else {
289
else {
249
   ( $tagid, $subfieldid ) =
290
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
250
            GetMarcFromKohaField( "biblio.biblionumber" );
291
    $tagid ||= "001";
251
	$tagid||="001";
252
}
292
}
253
293
294
my $sth_isbn;
254
# the SQL query to search on isbn
295
# the SQL query to search on isbn
255
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
296
if ($isbn_check) {
297
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
298
}
256
299
257
my $loghandle;
300
my $loghandle;
258
if ($logfile){
301
if ($logfile) {
259
   $loghandle= IO::File->new($logfile, $writemode) ;
302
    $loghandle= IO::File->new($logfile, $writemode);
260
   print $loghandle "id;operation;status\n";
303
    print $loghandle "id;operation;status\n";
261
}
304
}
262
305
306
my $record_number = 0;
263
my $logger = Koha::Logger->get;
307
my $logger = Koha::Logger->get;
264
my $schema = Koha::Database->schema;
308
my $schema = Koha::Database->schema;
265
$schema->txn_begin;
309
my $marc_records = [];
266
RECORD: while (  ) {
310
RECORD: while (  ) {
267
    my $record;
311
    my $record;
268
    # get records
312
    $record_number++;
313
    # get record
269
    eval { $record = $batch->next() };
314
    eval { $record = $batch->next() };
270
    if ( $@ ) {
315
    if ($@) {
271
        print "Bad MARC record $i: $@ skipped\n";
316
        print "Bad MARC record $record_number: $@ skipped\n";
272
        # FIXME - because MARC::Batch->next() combines grabbing the next
317
        # FIXME - because MARC::Batch->next() combines grabbing the next
273
        # blob and parsing it into one operation, a correctable condition
318
        # blob and parsing it into one operation, a correctable condition
274
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
319
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
Lines 277-555 RECORD: while ( ) { Link Here
277
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
322
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
278
        next;
323
        next;
279
    }
324
    }
280
    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
325
    if ($record) {
281
    last unless ( $record );
326
        # transcode the record to UTF8 if needed & applicable.
282
    $i++;
327
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
283
    if( ($verbose//1)==1 ) { #no dot for verbose==2
328
            my ($guessed_charset, $charset_errors);
284
        print "." . ( $i % 100==0 ? "\n$i" : '' );
329
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
330
            if ($guessed_charset eq 'failed') {
331
                warn "ERROR: failed to perform character conversion for record $record_number\n";
332
                next RECORD;
333
            }
334
        }
335
        SetUTF8Flag($record);
336
        &$localcust($record) if $localcust;
337
        push @{$marc_records}, $record;
338
    }
339
    else {
340
        last;
285
    }
341
    }
342
}
286
343
287
    # transcode the record to UTF8 if needed & applicable.
344
$record_number = 0;
288
    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
345
my $records_total = @{$marc_records};
289
        # FIXME update condition
346
$schema->txn_begin;
290
        my ($guessed_charset, $charset_errors);
347
RECORD: foreach my $record (@{$marc_records}) {
291
         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
348
    $record_number++;
292
        if ($guessed_charset eq 'failed') {
349
    if (($verbose//1) == 1) { #no dot for verbose==2
293
            warn "ERROR: failed to perform character conversion for record $i\n";
350
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
294
            next RECORD;            
295
        }
296
    }
351
    }
297
    SetUTF8Flag($record);
352
298
    if($marc_mod_template_id > 0) {
353
    if ($marc_mod_template_id > 0) {
299
    print "Modifying MARC\n" if $verbose;
354
        print "Modifying MARC\n" if $verbose;
300
    ModifyRecordWithTemplate( $marc_mod_template_id, $record );
355
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
301
    }
356
    }
302
    &$localcust($record) if $localcust;
357
303
    my $isbn;
358
    my $isbn;
304
    # remove trailing - in isbn (only for biblios, of course)
359
    # remove trailing - in isbn (only for biblios, of course)
305
    if( $biblios ) {
360
    if ($biblios && ($cleanisbn || $isbn_check)) {
306
        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
361
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
307
        my $field = $record->field($tag);
362
        my $field = $record->field($tag);
308
        $isbn = $field && $field->subfield('a');
363
        $isbn = $field && $field->subfield('a');
309
        if ( $isbn && $cleanisbn ) {
364
        if ($isbn && $cleanisbn) {
310
            $isbn =~ s/-//g;
365
            $isbn =~ s/-//g;
311
            $field->update('a' => $isbn);
366
            $field->update('a' => $isbn);
312
        }
367
        }
313
    }
368
    }
314
    my $id;
315
    # search for duplicates (based on Local-number)
369
    # search for duplicates (based on Local-number)
316
    my $originalid;
370
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
317
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
371
    my $matched_record_id = undef;
318
    if ($match) {
372
    if ($match) {
319
        require C4::Search;
373
        require C4::Search;
320
        my $query = build_query( $match, $record );
374
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
321
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
375
        my $query = build_query($match, $record);
322
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
376
        $logger->debug("Bulkmarcimport: $query");
377
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
323
        # changed to warn so able to continue with one broken record
378
        # changed to warn so able to continue with one broken record
324
        if ( defined $error ) {
379
        if (defined $error) {
325
            warn "unable to search the database for duplicates : $error";
380
            warn "unable to search the database for duplicates : $error";
326
            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
381
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
327
            next RECORD;
382
            next RECORD;
328
        }
383
        }
329
        if ( $results && scalar(@$results) == 1 ) {
384
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
330
            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
385
        # sub SimpleSearch could return undefined, but only on error, so
331
            SetUTF8Flag($marcrecord);
386
        # should not really need to safeguard here, but do so anyway
332
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
387
        $results //= [];
333
            if ( $authorities && $marcFlavour ) {
388
        if (@{$results} == 1) {
334
                #Skip if authority in database is the same as the on in database
389
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
335
                if ( $marcrecord->field('005') && $record->field('005') &&
390
            SetUTF8Flag($matched_record);
336
                     $marcrecord->field('005')->data && $record->field('005')->data &&
391
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
337
                     $marcrecord->field('005')->data >= $record->field('005')->data ) {
392
393
            if ($authorities && $marc_flavour) {
394
                #Skip if authority in database is the same or newer than the incoming record
395
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
338
                    if ($yamlfile) {
396
                    if ($yamlfile) {
339
                        $yamlhash->{$originalid}->{'authid'} = $id;
397
                        $yamlhash->{$originalid} = YAMLFileEntry(
340
398
                            $matched_record,
341
                        # we recover all subfields of the heading authorities
399
                            $matched_record_id,
342
                        my @subfields;
400
                            0
343
                        foreach my $field ( $marcrecord->field("2..") ) {
401
                        );
344
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
345
                        }
346
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
347
                        $yamlhash->{$originalid}->{'updated'} = 0;
348
                    }
402
                    }
349
                    next;
403
                    next;
350
                }
404
                }
351
            }
405
            }
352
        } elsif ( $results && scalar(@$results) > 1 ) {
353
            $logger->debug("more than one match for $query");
354
        } else {
355
            $logger->debug("nomatch for $query");
356
        }
406
        }
357
    }
407
        elsif(@{$results} > 1) {
358
    if ($keepids && $originalid) {
408
            $logger->debug("More than one match for: $query");
409
        }
410
        else {
411
            $logger->debug("No match for: $query");
412
        }
413
414
        if ($keepids && $originalid) {
359
            my $storeidfield;
415
            my $storeidfield;
360
            if ( length($keepids) == 3 ) {
416
            if (length($keepids) == 3) {
361
                $storeidfield = MARC::Field->new( $keepids, $originalid );
417
                $storeidfield = MARC::Field->new($keepids, $originalid);
362
            } else {
418
            } else {
363
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
419
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
364
            }
420
            }
365
            $record->insert_fields_ordered($storeidfield);
421
            $record->insert_fields_ordered($storeidfield);
366
            $record->delete_field( $record->field($tagid) );
422
            $record->delete_field($record->field($tagid));
423
        }
367
    }
424
    }
425
368
    foreach my $stringfilter (@$filters) {
426
    foreach my $stringfilter (@$filters) {
369
        if ( length($stringfilter) == 3 ) {
427
        if (length($stringfilter) == 3) {
370
            foreach my $field ( $record->field($stringfilter) ) {
428
            foreach my $field ($record->field($stringfilter)) {
371
                $record->delete_field($field);
429
                $record->delete_field($field);
372
                $logger->debug("removed : ", $field->as_string);
430
                $logger->debug("Removed: ", $field->as_string);
373
            }
431
            }
374
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
432
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
375
            my $removetag = $1;
433
            my $removetag = $1;
376
            my $removesubfield = $2;
434
            my $removesubfield = $2;
377
            my $removematch = $3;
435
            my $removematch = $3;
378
            if ( ( $removetag > "010" ) && $removesubfield ) {
436
            if (($removetag > "010") && $removesubfield) {
379
                foreach my $field ( $record->field($removetag) ) {
437
                foreach my $field ($record->field($removetag)) {
380
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
438
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
381
                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));
439
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
382
                }
440
                }
383
            }
441
            }
384
        }
442
        }
385
    }
443
    }
386
    unless ($test_parameter) {
444
    unless ($test_parameter) {
387
        if ($authorities){
445
        if ($authorities) {
388
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
446
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
389
            my $authid= ($id?$id:GuessAuthId($record));
447
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
390
            if ($authid && GetAuthority($authid) && $update ){
448
            if ($authid && GetAuthority($authid) && $update) {
391
            ## Authority has an id and is in database : Replace
449
            ## Authority has an id and is in database : Replace
392
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
450
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
393
                if ($@){
451
                if ($@) {
394
                    warn "Problem with authority $authid Cannot Modify";
452
                    warn "Problem with authority $authid Cannot Modify";
395
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
453
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
396
                }
454
                }
397
				else{
455
                else{
398
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
456
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
399
				}
457
                }
400
            }  
458
            }
401
	        else {
459
            else {
402
            ## True insert in database
460
            ## True insert in database
403
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
461
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
404
                if ($@){
462
                if ($@) {
405
                    warn "Problem with authority $authid Cannot Add".$@;
463
                    warn "Problem with authority $originalid Cannot Add ".$@;
406
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
464
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
407
                }
465
                }
408
   				else{
466
                else {
409
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
467
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
410
				}
468
                }
411
 	        }
469
412
            if ($yamlfile) {
413
            $yamlhash->{$originalid}->{'authid'} = $authid;
414
            my @subfields;
415
            foreach my $field ( $record->field("2..") ) {
416
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
417
            }
470
            }
418
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
471
            if ($yamlfile) {
419
            $yamlhash->{$originalid}->{'updated'} = 1;
472
                $yamlhash->{$originalid} = YAMLFileEntry(
473
                    $record,
474
                    $authid,
475
                    1 #@FIXME: Really always updated?
476
                );
420
            }
477
            }
421
        }
478
        }
422
        else {
479
        else {
423
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
480
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
424
            $biblionumber = $id;
425
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
481
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
426
            if (!$biblionumber && $isbn_check && $isbn) {
482
            if (!$matched_record_id && $isbn_check && $isbn) {
427
    #         warn "search ISBN : $isbn";
428
                $sth_isbn->execute($isbn);
483
                $sth_isbn->execute($isbn);
429
                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
484
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
485
            }
486
487
            if (defined $idmapfl && $matched_record_id) {
488
                if ($sourcetag < "010") {
489
                    if ($record->field($sourcetag)) {
490
                        my $source = $record->field($sourcetag)->data();
491
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
492
                    }
493
                }
494
                else {
495
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
496
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
497
                }
430
            }
498
            }
431
        	if (defined $idmapfl) {
499
432
			 	if ($sourcetag < "010"){
500
            # create biblio, unless we already have it ( either match or isbn )
433
					if ($record->field($sourcetag)){
501
            if ($matched_record_id) {
434
					  my $source = $record->field($sourcetag)->data();
502
                # TODO: Implement also for authority records!
435
                      printf($idmapfh "%s|%s\n",$source,$biblionumber);
436
					}
437
			    } else {
438
					my $source=$record->subfield($sourcetag,$sourcesubfield);
439
                    printf($idmapfh "%s|%s\n",$source,$biblionumber);
440
			  }
441
			}
442
					# create biblio, unless we already have it ( either match or isbn )
443
            if ($biblionumber) {
444
                eval{
503
                eval{
445
                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
504
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
446
                };
505
                };
447
                if ($update) {
506
                if ($update) {
448
                    eval { ModBiblio( $record, $biblionumber, $framework, { overlay_context => { source => 'bulkmarcimport' } } ) };
507
                    my $success;
508
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
449
                    if ($@) {
509
                    if ($@) {
450
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
510
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
451
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
511
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
452
                        next RECORD;
512
                        next RECORD;
453
                    } else {
454
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
455
                    }
513
                    }
456
                } else {
514
                    elsif (!$success) {
457
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
515
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
458
                }
516
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
459
            } else {
460
                if ($insert) {
461
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
462
                    if ($@) {
463
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
464
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
465
                        next RECORD;
517
                        next RECORD;
466
                    } else {
467
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
468
                    }
518
                    }
469
                } else {
519
                    else {
470
                    warn "WARNING: Updating record ".($id||$originalid)." failed";
520
                        $record_id = $matched_record_id;
471
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
521
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
522
                    }
523
                }
524
                else {
525
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
526
                }
527
            }
528
            elsif ($insert) {
529
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
530
                if ($@) {
531
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
532
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
472
                    next RECORD;
533
                    next RECORD;
473
                }
534
                }
535
                else {
536
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
537
                }
474
            }
538
            }
475
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
539
            else {
476
            my $error_adding = $@;
540
                warn "WARNING: Updating record ".($originalid)." failed";
477
            # Work on a clone so that if there are real errors, we can maybe
541
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
478
            # fix them up later.
479
			my $clone_record = $record->clone();
480
            C4::Biblio::_strip_item_fields($clone_record, '');
481
            # This sets the marc fields if there was an error, and also calls
482
            # defer_marc_save.
483
            ModBiblioMarc( $clone_record, $biblionumber );
484
            if ( $error_adding ) {
485
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
486
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
487
                # if we failed because of an exception, assume that 
488
                # the MARC columns in biblioitems were not set.
489
                next RECORD;
542
                next RECORD;
490
            }
543
            }
491
 			else{
544
            my $record_has_added_items = 0;
492
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
545
            if ($record_id) {
493
			}
546
                $yamlhash->{$originalid} = $record_id if $yamlfile;
494
            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
547
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
495
                # Find the record called 'barcode'
548
                $record_has_added_items = @{$itemnumbers_ref};
496
                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
549
                my $error_adding = $@;
497
                # Now remove any items that didn't have a duplicate_barcode error,
550
                # Work on a clone so that if there are real errors, we can maybe
498
                # erase the barcodes on items that did, and re-add those items.
551
                # fix them up later.
499
                my %dupes;
552
                my $clone_record = $record->clone();
500
                foreach my $i (0 .. $#{$errors_ref}) {
553
                C4::Biblio::_strip_item_fields($clone_record, $framework);
501
                    my $ref = $errors_ref->[$i];
554
                # This sets the marc fields if there was an error, and also calls
502
                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
555
                # defer_marc_save.
503
                        $dupes{$ref->{item_sequence}} = 1;
556
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
504
                        # Delete the error message because we're going to
557
                if ($error_adding) {
505
                        # retry this one.
558
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
506
                        delete $errors_ref->[$i];
559
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
507
                    }
508
                }
509
                my $seq = 0;
510
                foreach my $field ($record->field($tag)) {
511
                    $seq++;
512
                    if ($dupes{$seq}) {
513
                        # Here we remove the barcode
514
                        $field->delete_subfield(code => $sub);
515
                    } else {
516
                        # otherwise we delete the field because we don't want
517
                        # two of them
518
                        $record->delete_fields($field);
519
                    }
520
                }
521
                # Now re-add the record as before, adding errors to the prev list
522
                my $more_errors;
523
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
524
                if ( $@ ) {
525
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
526
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
527
                    # if we failed because of an exception, assume that
560
                    # if we failed because of an exception, assume that
528
                    # the MARC columns in biblioitems were not set.
561
                    # the MARC columns in biblioitems were not set.
529
                    ModBiblioMarc( $record, $biblionumber );
530
                    next RECORD;
562
                    next RECORD;
531
                } else {
532
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
533
                }
563
                }
534
                push @$errors_ref, @{ $more_errors };
564
                else {
535
            }
565
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
536
            if ($#{ $errors_ref } > -1) {
566
                }
537
                report_item_errors($biblionumber, $errors_ref);
567
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
568
                    # Find the record called 'barcode'
569
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
570
                    # Now remove any items that didn't have a duplicate_barcode error,
571
                    # erase the barcodes on items that did, and re-add those items.
572
                    my %dupes;
573
                    foreach my $i (0 .. $#{$errors_ref}) {
574
                        my $ref = $errors_ref->[$i];
575
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
576
                            $dupes{$ref->{item_sequence}} = 1;
577
                            # Delete the error message because we're going to
578
                            # retry this one.
579
                            delete $errors_ref->[$i];
580
                        }
581
                    }
582
                    my $seq = 0;
583
                    foreach my $field ($record->field($tag)) {
584
                        $seq++;
585
                        if ($dupes{$seq}) {
586
                            # Here we remove the barcode
587
                            $field->delete_subfield(code => $sub);
588
                        }
589
                        else {
590
                            # otherwise we delete the field because we don't want
591
                            # two of them
592
                            $record->delete_fields($field);
593
                        }
594
                    }
595
                    # Now re-add the record as before, adding errors to the prev list
596
                    my $more_errors;
597
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
598
                    $record_has_added_items ||= @{$itemnumbers_ref};
599
                    if ($@) {
600
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
601
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
602
                        # if we failed because of an exception, assume that
603
                        # the MARC columns in biblioitems were not set.
604
605
                        # @FIXME: Why do we save here without stripping items? Besides,
606
                        # save with stripped items has already been performed
607
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
608
                        next RECORD;
609
                    }
610
                    else {
611
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
612
                    }
613
                    push @$errors_ref, @{$more_errors};
614
                }
615
                if (@{$errors_ref}) {
616
                    report_item_errors($record_id, $errors_ref);
617
                }
618
                C4::Biblio::_strip_item_fields($record, $framework);
619
                if ($record_has_added_items || $matched_record_id) {
620
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
621
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
622
                }
623
                push @search_engine_record_ids, $record_id;
624
                push @search_engine_records, $record;
538
            }
625
            }
539
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
540
        }
626
        }
541
        if ( 0 == $i % $commitnum ) {
627
        if ($record_number % $commitnum == 0 || $record_number == $number || $record_number == $records_total) {
542
            $schema->txn_commit;
628
            $schema->txn_commit;
543
            $schema->txn_begin;
629
            $schema->txn_begin;
630
            if ($indexer) {
631
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
632
                @search_engine_record_ids = ();
633
                @search_engine_records = ();
634
            }
544
        }
635
        }
545
    }
636
    }
546
    print $record->as_formatted()."\n" if ($verbose//0)==2;
637
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
547
    last if $i == $number;
638
    last if $record_number == $number;
548
}
639
}
549
$schema->txn_commit;
640
$schema->txn_commit;
550
641
551
if ($fk_off) {
642
if ($fk_off) {
552
	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
643
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
553
}
644
}
554
645
555
# Restore CataloguingLog and AuthoritiesLog
646
# Restore CataloguingLog and AuthoritiesLog
Lines 557-567 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; Link Here
557
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
648
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
558
649
559
my $timeneeded = gettimeofday - $starttime;
650
my $timeneeded = gettimeofday - $starttime;
560
print "\n$i MARC records done in $timeneeded seconds\n";
651
print "\n$record_number MARC records done in $timeneeded seconds\n";
561
if ($logfile){
652
if ($logfile) {
562
  print $loghandle "file : $input_marc_file\n";
653
    print $loghandle "file : $input_marc_file\n";
563
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
654
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
564
  $loghandle->close;
655
    $loghandle->close;
565
}
656
}
566
if ($yamlfile) {
657
if ($yamlfile) {
567
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
658
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
Lines 569-615 if ($yamlfile) { Link Here
569
}
660
}
570
exit 0;
661
exit 0;
571
662
572
sub GetRecordId{
663
sub YAMLFileEntry {
573
	my $marcrecord=shift;
664
    my ($record, $record_id, $updated) = @_;
574
	my $tag=shift;
665
575
	my $subfield=shift;
666
    my $entry = {
576
	my $id;
667
        authid => $record_id
577
	if ($tag lt "010"){
668
    };
578
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
669
579
	} 
670
    # we recover all subfields of the heading authorities
580
	elsif ($subfield){
671
    my @subfields;
581
		if ($marcrecord->field($tag)){
672
    foreach my $field ($record->field("2..")) {
582
			return $marcrecord->subfield($tag,$subfield);
673
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
583
		}
674
    }
584
	}
675
    $entry->{'subfields'} = \@subfields;
585
	return $id;
676
    $entry->{'updated'} = $updated;
677
678
    return $entry;
586
}
679
}
587
sub build_query {
680
588
	my $match = shift;
681
sub RecordRevisionIsGtOrEq {
589
	my $record=shift;
682
    my ($record_a, $record_b) = @_;
590
        my @searchstrings;
683
    return $record_a->field('005') && $record_b->field('005') &&
591
	foreach my $matchingpoint (@$match){
684
    $record_a->field('005')->data && $record_b->field('005')->data &&
592
	  my $string = build_simplequery($matchingpoint,$record);
685
    $record_a->field('005')->data >= $record_b->field('005')->data;
593
	  push @searchstrings,$string if (length($string)>0);
686
}
687
688
sub GetRecordId {
689
    my $marcrecord = shift;
690
    my $tag = shift;
691
    my $subfield = shift;
692
    if ($tag lt "010") {
693
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
694
    }
695
    elsif ($subfield) {
696
        if ($marcrecord->field($tag)) {
697
            return $marcrecord->subfield($tag, $subfield);
594
        }
698
        }
699
    }
700
    return undef;
701
}
702
sub build_query {
703
    my ($match, $record) = @_;
704
    my @searchstrings;
705
706
    foreach my $matchpoint (@$match) {
707
        my $query = build_simplequery($matchpoint, $record);
708
        push (@searchstrings, $query) if $query;
709
    }
595
    my $op = 'AND';
710
    my $op = 'AND';
596
    return join(" $op ",@searchstrings);
711
    return join(" $op ", @searchstrings);
597
}
712
}
598
sub build_simplequery {
713
sub build_simplequery {
599
	my $element=shift;
714
    my ($matchpoint, $record) = @_;
600
	my $record=shift;
715
601
    my @searchstrings;
716
    my @searchstrings;
602
    my ($index,$recorddata)=split /,/,$element;
717
    my ($index, $record_data) = split (/,/, $matchpoint);
603
    if ($recorddata=~/(\d{3})(.*)/) {
718
    if ($record_data =~ /(\d{3})(.*)/) {
604
        my ($tag,$subfields) =($1,$2);
719
        my ($tag, $subfields) = ($1, $2);
605
        foreach my $field ($record->field($tag)){
720
        foreach my $field ($record->field($tag)) {
606
		  if (length($field->as_string("$subfields"))>0){
721
            if (length($field->as_string("$subfields")) > 0) {
607
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
722
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
608
		  }
723
            }
609
        }
724
        }
610
    }
725
    }
726
    else {
727
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
728
    }
611
    my $op = 'AND';
729
    my $op = 'AND';
612
    return join(" $op ",@searchstrings);
730
    return join(" $op ", @searchstrings);
613
}
731
}
614
sub report_item_errors {
732
sub report_item_errors {
615
    my $biblionumber = shift;
733
    my $biblionumber = shift;
Lines 624-643 sub report_item_errors { Link Here
624
        print $msg, "\n";
742
        print $msg, "\n";
625
    }
743
    }
626
}
744
}
627
sub printlog{
745
sub printlog {
628
	my $logelements=shift;
746
    my $logelements = shift;
629
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
747
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
630
}
748
}
631
sub get_heading_fields{
749
sub get_heading_fields {
632
    my $headingfields;
750
    my $headingfields;
633
    if ($authtypes){
751
    if ($authtypes) {
634
        $headingfields = YAML::XS::LoadFile($authtypes);
752
        $headingfields = YAML::XS::LoadFile($authtypes);
635
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
753
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
636
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
754
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
637
    }
755
    }
638
    unless ($headingfields){
756
    unless ($headingfields) {
639
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
757
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
640
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
758
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
641
    }
759
    }
642
    return $headingfields;
760
    return $headingfields;
643
}
761
}
644
- 

Return to bug 29440