View | Details | Raw Unified | Return to bug 29440
Collapse All | Expand All

(-)a/C4/Biblio.pm (-7 / +28 lines)
Lines 310-316 sub AddBiblio { Link Here
310
310
311
=head2 ModBiblio
311
=head2 ModBiblio
312
312
313
  ModBiblio( $record,$biblionumber,$frameworkcode, $disable_autolink);
313
  ModBiblio($record, $biblionumber, $frameworkcode, $options);
314
314
315
Replace an existing bib record identified by C<$biblionumber>
315
Replace an existing bib record identified by C<$biblionumber>
316
with one supplied by the MARC::Record object C<$record>.  The embedded
316
with one supplied by the MARC::Record object C<$record>.  The embedded
Lines 326-341 in the C<biblio> and C<biblioitems> tables, as well as Link Here
326
which fields are used to store embedded item, biblioitem,
326
which fields are used to store embedded item, biblioitem,
327
and biblionumber data for indexing.
327
and biblionumber data for indexing.
328
328
329
Unless C<$disable_autolink> is passed ModBiblio will relink record headings
329
The C<$options> argument is a hashref with additional parameters:
330
331
=over 4
332
333
=item C<context>
334
335
This parameter is forwared to L</ApplyMarcMergeRules> where it is used for
336
selecting the current rule set if Marc Merge Rules is enabled.
337
See L</ApplyMarcMergeRules> for more details.
338
339
=item C<disable_autolink>
340
341
Unless C<disable_autolink> is passed ModBiblio will relink record headings
330
to authorities based on settings in the system preferences. This flag allows
342
to authorities based on settings in the system preferences. This flag allows
331
us to not relink records when the authority linker is saving modifications.
343
us to not relink records when the authority linker is saving modifications.
332
344
345
=back
346
333
Returns 1 on success 0 on failure
347
Returns 1 on success 0 on failure
334
348
335
=cut
349
=cut
336
350
337
sub ModBiblio {
351
sub ModBiblio {
338
    my ( $record, $biblionumber, $frameworkcode, $disable_autolink ) = @_;
352
    my ( $record, $biblionumber, $frameworkcode, $options ) = @_;
353
    my %mod_biblio_marc_options;
354
    $mod_biblio_marc_options{'defer_search_engine_indexing'} =
355
        defined $options && exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
356
339
    if (!$record) {
357
    if (!$record) {
340
        carp 'No record passed to ModBiblio';
358
        carp 'No record passed to ModBiblio';
341
        return 0;
359
        return 0;
Lines 346-352 sub ModBiblio { Link Here
346
        logaction( "CATALOGUING", "MODIFY", $biblionumber, "biblio BEFORE=>" . $newrecord->as_formatted );
364
        logaction( "CATALOGUING", "MODIFY", $biblionumber, "biblio BEFORE=>" . $newrecord->as_formatted );
347
    }
365
    }
348
366
349
    if ( !$disable_autolink && C4::Context->preference('BiblioAddsAuthorities') ) {
367
    if ( !$options->{disable_autolink} && C4::Context->preference('BiblioAddsAuthorities') ) {
350
        BiblioAutoLink( $record, $frameworkcode );
368
        BiblioAutoLink( $record, $frameworkcode );
351
    }
369
    }
352
370
Lines 383-389 sub ModBiblio { Link Here
383
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
401
    _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode );
384
402
385
    # update the MARC record (that now contains biblio and items) with the new record data
403
    # update the MARC record (that now contains biblio and items) with the new record data
386
    &ModBiblioMarc( $record, $biblionumber );
404
    ModBiblioMarc( $record, $biblionumber, \%mod_biblio_marc_options );
387
405
388
    # modify the other koha tables
406
    # modify the other koha tables
389
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
407
    _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode );
Lines 2916-2922 sub _koha_delete_biblio_metadata { Link Here
2916
2934
2917
=head2 ModBiblioMarc
2935
=head2 ModBiblioMarc
2918
2936
2919
  &ModBiblioMarc($newrec,$biblionumber);
2937
  &ModBiblioMarc($newrec,$biblionumber,$options);
2920
2938
2921
Add MARC XML data for a biblio to koha
2939
Add MARC XML data for a biblio to koha
2922
2940
Lines 2927-2933 Function exported, but should NOT be used, unless you really know what you're do Link Here
2927
sub ModBiblioMarc {
2945
sub ModBiblioMarc {
2928
    # pass the MARC::Record to this function, and it will create the records in
2946
    # pass the MARC::Record to this function, and it will create the records in
2929
    # the marcxml field
2947
    # the marcxml field
2930
    my ( $record, $biblionumber ) = @_;
2948
    my ( $record, $biblionumber, $options ) = @_;
2949
    my $defer_search_engine_indexing =
2950
        defined $options && exists $options->{'defer_search_engine_indexing'} && $options->{'defer_search_engine_indexing'};
2951
2931
    if ( !$record ) {
2952
    if ( !$record ) {
2932
        carp 'ModBiblioMarc passed an undefined record';
2953
        carp 'ModBiblioMarc passed an undefined record';
2933
        return;
2954
        return;
(-)a/misc/link_bibs_to_authorities.pl (-1 / +1 lines)
Lines 233-239 sub process_bib { Link Here
233
            );
233
            );
234
        }
234
        }
235
        if ( not $test_only ) {
235
        if ( not $test_only ) {
236
            ModBiblio( $bib, $biblionumber, $frameworkcode, 1 );
236
            ModBiblio( $bib, $biblionumber, $frameworkcode, { disable_autolink => 1 });
237
            #Last param is to note ModBiblio was called from linking script and bib should not be linked again
237
            #Last param is to note ModBiblio was called from linking script and bib should not be linked again
238
            $num_bibs_modified++;
238
            $num_bibs_modified++;
239
        }
239
        }
(-)a/misc/migration_tools/bulkmarcimport.pl (-307 / +431 lines)
Lines 23-28 use C4::Biblio qw( Link Here
23
    GetMarcFromKohaField
23
    GetMarcFromKohaField
24
    ModBiblio
24
    ModBiblio
25
    ModBiblioMarc
25
    ModBiblioMarc
26
    GetFrameworkCode
27
    GetMarcBiblio
26
);
28
);
27
use C4::Koha;
29
use C4::Koha;
28
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
30
use C4::Charset qw( MarcToUTF8Record SetUTF8Flag );
Lines 38-66 use Time::HiRes qw( gettimeofday ); Link Here
38
use Getopt::Long qw( GetOptions );
40
use Getopt::Long qw( GetOptions );
39
use IO::File;
41
use IO::File;
40
use Pod::Usage qw( pod2usage );
42
use Pod::Usage qw( pod2usage );
43
use List::MoreUtils qw(any);
41
44
42
use Koha::Logger;
45
use Koha::Logger;
43
use Koha::Biblios;
46
use Koha::Biblios;
44
use Koha::SearchEngine;
47
use Koha::SearchEngine;
45
use Koha::SearchEngine::Search;
48
use Koha::SearchEngine::Search;
49
use Koha::Plugins::Handler;
46
50
47
use open qw( :std :encoding(UTF-8) );
51
use open qw( :std :encoding(UTF-8) );
48
binmode( STDOUT, ":encoding(UTF-8)" );
52
binmode(STDOUT, ":encoding(UTF-8)");
49
my ( $input_marc_file, $number, $offset) = ('',0,0);
53
my ($input_marc_file, $number, $offset, $cleanisbn) = ('', 0, 0, 1);
50
my ($version, $delete, $test_parameter, $skip_marc8_conversion, $char_encoding, $verbose, $commit, $fk_off,$format,$biblios,$authorities,$keepids,$match, $isbn_check, $logfile);
54
my $version;
51
my ( $insert, $filters, $update, $all, $yamlfile, $authtypes, $append );
55
my $delete;
52
my $cleanisbn = 1;
56
my $test_parameter;
53
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
57
my $skip_marc8_conversion;
58
my $char_encoding;
59
my $verbose;
60
my $commit;
61
my $fk_off;
62
my $format;
63
my $biblios;
64
my $authorities;
65
my $keepids;
66
my $match;
67
my $isbn_check;
68
my $logfile;
69
my $insert;
70
my $filters;
71
my $update;
72
my $all;
73
my $yamlfile;
74
my $authtypes;
75
my $append;
76
my $sourcetag;
77
my $sourcesubfield;
78
my $idmapfl;
79
my $dedup_barcode;
54
my $framework = '';
80
my $framework = '';
55
my $localcust;
81
my $localcust;
56
my $marc_mod_template = '';
82
my $marc_mod_template = '';
57
my $marc_mod_template_id = -1;
83
my $marc_mod_template_id = -1;
58
84
$| = 1;
59
$|=1;
60
85
61
GetOptions(
86
GetOptions(
62
    'commit:f'    => \$commit,
87
    'commit:f' => \$commit,
63
    'file:s'    => \$input_marc_file,
88
    'file:s' => \$input_marc_file,
64
    'n:f' => \$number,
89
    'n:f' => \$number,
65
    'o|offset:f' => \$offset,
90
    'o|offset:f' => \$offset,
66
    'h' => \$version,
91
    'h' => \$version,
Lines 77-98 GetOptions( Link Here
77
    'b|biblios' => \$biblios,
102
    'b|biblios' => \$biblios,
78
    'a|authorities' => \$authorities,
103
    'a|authorities' => \$authorities,
79
    'authtypes:s' => \$authtypes,
104
    'authtypes:s' => \$authtypes,
80
    'filter=s@'     => \$filters,
105
    'filter=s@' => \$filters,
81
    'insert'        => \$insert,
106
    'insert' => \$insert,
82
    'update'        => \$update,
107
    'update' => \$update,
83
    'all'           => \$all,
108
    'all' => \$all,
84
    'match=s@'    => \$match,
109
    'match=s@' => \$match,
85
    'i|isbn' => \$isbn_check,
110
    'i|isbn' => \$isbn_check,
86
    'x:s' => \$sourcetag,
111
    'x:s' => \$sourcetag,
87
    'y:s' => \$sourcesubfield,
112
    'y:s' => \$sourcesubfield,
88
    'idmap:s' => \$idmapfl,
113
    'idmap:s' => \$idmapfl,
89
    'cleanisbn!'     => \$cleanisbn,
114
    'cleanisbn!' => \$cleanisbn,
90
    'yaml:s'        => \$yamlfile,
115
    'yaml:s' => \$yamlfile,
91
    'dedupbarcode' => \$dedup_barcode,
116
    'dedupbarcode' => \$dedup_barcode,
92
    'framework=s' => \$framework,
117
    'framework=s' => \$framework,
93
    'custom:s'    => \$localcust,
118
    'custom:s' => \$localcust,
94
    'marcmodtemplate:s' => \$marc_mod_template,
119
    'marcmodtemplate:s' => \$marc_mod_template,
95
);
120
);
121
96
$biblios ||= !$authorities;
122
$biblios ||= !$authorities;
97
$insert  ||= !$update;
123
$insert  ||= !$update;
98
my $writemode = ($append) ? "a" : "w";
124
my $writemode = ($append) ? "a" : "w";
Lines 104-109 if ($all) { Link Here
104
    $update = 1;
130
    $update = 1;
105
}
131
}
106
132
133
my $using_elastic_search = (C4::Context->preference('SearchEngine') eq 'Elasticsearch');
134
my $modify_biblio_marc_options = {
135
    'defer_search_engine_indexing' => $using_elastic_search,
136
};
137
138
my @search_engine_record_ids;
139
my @search_engine_records;
140
my $indexer;
141
if ($using_elastic_search) {
142
    use Koha::SearchEngine::Elasticsearch::Indexer;
143
    $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
144
        { index => $authorities ?
145
            $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX :
146
            $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX
147
        }
148
    );
149
}
150
107
if ($version || ($input_marc_file eq '')) {
151
if ($version || ($input_marc_file eq '')) {
108
    pod2usage( -verbose => 2 );
152
    pod2usage( -verbose => 2 );
109
    exit;
153
    exit;
Lines 112-124 if( $update && !( $match || $isbn_check ) ) { Link Here
112
    warn "Using -update without -match or -isbn seems to be useless.\n";
156
    warn "Using -update without -match or -isbn seems to be useless.\n";
113
}
157
}
114
158
115
if(defined $localcust) { #local customize module
159
if (defined $localcust) { #local customize module
116
    if(!-e $localcust) {
160
    if (!-e $localcust) {
117
        $localcust= $localcust||'LocalChanges'; #default name
161
        $localcust = $localcust || 'LocalChanges'; #default name
118
        $localcust=~ s/^.*\/([^\/]+)$/$1/; #extract file name only
162
        $localcust =~ s/^.*\/([^\/]+)$/$1/; #extract file name only
119
        $localcust=~ s/\.pm$//;           #remove extension
163
        $localcust =~ s/\.pm$//;           #remove extension
120
        my $fqcust= $FindBin::Bin."/$localcust.pm"; #try migration_tools dir
164
        my $fqcust = $FindBin::Bin . "/$localcust.pm"; #try migration_tools dir
121
        if(-e $fqcust) {
165
        if (-e $fqcust) {
122
            $localcust= $fqcust;
166
            $localcust= $fqcust;
123
        }
167
        }
124
        else {
168
        else {
Lines 127-133 if(defined $localcust) { #local customize module Link Here
127
        }
171
        }
128
    }
172
    }
129
    require $localcust if $localcust;
173
    require $localcust if $localcust;
130
    $localcust=\&customize if $localcust;
174
    $localcust = \&customize if $localcust;
131
}
175
}
132
176
133
if($marc_mod_template ne '') {
177
if($marc_mod_template ne '') {
Lines 152-219 if($marc_mod_template ne '') { Link Here
152
}
196
}
153
197
154
my $dbh = C4::Context->dbh;
198
my $dbh = C4::Context->dbh;
155
my $heading_fields=get_heading_fields();
199
my $heading_fields = get_heading_fields();
156
157
my $idmapfh;
200
my $idmapfh;
201
158
if (defined $idmapfl) {
202
if (defined $idmapfl) {
159
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
203
  open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
160
}
204
}
161
205
162
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
206
if ((not defined $sourcesubfield) && (not defined $sourcetag)) {
163
  $sourcetag="910";
207
    $sourcetag = "910";
164
  $sourcesubfield="a";
208
    $sourcesubfield = "a";
165
}
209
}
166
210
167
168
# Disable logging for the biblios and authorities import operation. It would unnecessarily
211
# Disable logging for the biblios and authorities import operation. It would unnecessarily
169
# slow the import
212
# slow the import
170
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
213
$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
171
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
214
$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
172
215
173
if ($fk_off) {
216
if ($fk_off) {
174
	$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
217
    $dbh->do("SET FOREIGN_KEY_CHECKS = 0");
175
}
218
}
176
219
177
178
if ($delete) {
220
if ($delete) {
179
	if ($biblios){
221
    if ($biblios) {
180
    	print "deleting biblios\n";
222
        print "Deleting biblios\n";
181
        $dbh->do("DELETE FROM biblio");
223
        $dbh->do("DELETE FROM biblio");
182
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
224
        $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
183
        $dbh->do("DELETE FROM biblioitems");
225
        $dbh->do("DELETE FROM biblioitems");
184
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
226
        $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
185
        $dbh->do("DELETE FROM items");
227
        $dbh->do("DELETE FROM items");
186
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
228
        $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
187
	}
229
    }
188
	else {
230
    else {
189
    	print "deleting authorities\n";
231
        print "Deleting authorities\n";
190
    	$dbh->do("truncate auth_header");
232
        $dbh->do("truncate auth_header");
191
	}
233
    }
192
    $dbh->do("truncate zebraqueue");
234
    $dbh->do("truncate zebraqueue");
193
}
235
}
194
236
195
196
197
if ($test_parameter) {
237
if ($test_parameter) {
198
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
238
    print "TESTING MODE ONLY\n    DOING NOTHING\n===============\n";
199
}
239
}
200
240
201
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
241
my $batch;
242
my $marc_flavour = C4::Context->preference('marcflavour') || 'MARC21';
202
243
203
# The definition of $searcher must be before MARC::Batch->new
244
# The definition of $searcher must be before MARC::Batch->new
204
my $searcher = Koha::SearchEngine::Search->new(
245
my $searcher = Koha::SearchEngine::Search->new(
205
    {
246
    {
206
        index => (
247
        index => (
207
              $authorities
248
            $authorities
208
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
249
            ? $Koha::SearchEngine::AUTHORITIES_INDEX
209
            : $Koha::SearchEngine::BIBLIOS_INDEX
250
            : $Koha::SearchEngine::BIBLIOS_INDEX
210
        )
251
        )
211
    }
252
    }
212
);
253
);
213
254
214
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
255
print "Characteristic MARC flavour: $marc_flavour\n" if $verbose;
215
my $starttime = gettimeofday;
256
my $starttime = gettimeofday;
216
my $batch;
257
217
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
258
my $fh = IO::File->new($input_marc_file); # don't let MARC::Batch open the file, as it applies the ':utf8' IO layer
218
if (defined $format && $format =~ /XML/i) {
259
if (defined $format && $format =~ /XML/i) {
219
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
260
    # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
Lines 226-279 if (defined $format && $format =~ /XML/i) { Link Here
226
    #       extract the records, not using regexes to look
267
    #       extract the records, not using regexes to look
227
    #       for <record>.*</record>.
268
    #       for <record>.*</record>.
228
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
269
    $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
229
    my $recordformat= ($marcFlavour eq "MARC21"?"USMARC":uc($marcFlavour));
270
    my $recordformat = ($marc_flavour eq "MARC21" ? "USMARC" : uc($marc_flavour));
230
#UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
271
    #UNIMARC Authorities have a different way to manage encoding than UNIMARC biblios.
231
    $recordformat=$recordformat."AUTH" if ($authorities and $marcFlavour ne "MARC21");
272
    $recordformat = $recordformat . "AUTH" if ($authorities and $marc_flavour ne "MARC21");
232
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
273
    $MARC::File::XML::_load_args{RecordFormat} = $recordformat;
233
    $batch = MARC::Batch->new( 'XML', $fh );
274
    $batch = MARC::Batch->new('XML', $fh);
234
} else {
275
}
235
    $batch = MARC::Batch->new( 'USMARC', $fh );
276
else {
277
    $batch = MARC::Batch->new('USMARC', $fh);
236
}
278
}
279
237
$batch->warnings_off();
280
$batch->warnings_off();
238
$batch->strict_off();
281
$batch->strict_off();
239
my $i=0;
240
my $commitnum = $commit ? $commit : 50;
282
my $commitnum = $commit ? $commit : 50;
241
my $yamlhash;
283
my $yamlhash;
242
284
243
# Skip file offset
285
# Skip file offset
244
if ( $offset ) {
286
if ($offset) {
245
    print "Skipping file offset: $offset records\n";
287
    print "Skipping file offset: $offset records\n";
246
    $batch->next() while ($offset--);
288
    $batch->next() while ($offset--);
247
}
289
}
248
290
249
my ($tagid,$subfieldid);
291
my ($tagid, $subfieldid);
250
if ($authorities){
292
if ($authorities) {
251
	  $tagid='001';
293
    $tagid = '001';
252
}
294
}
253
else {
295
else {
254
   ( $tagid, $subfieldid ) =
296
    ($tagid, $subfieldid) = GetMarcFromKohaField("biblio.biblionumber");
255
            GetMarcFromKohaField( "biblio.biblionumber" );
297
    $tagid ||= "001";
256
	$tagid||="001";
257
}
298
}
258
299
300
my $sth_isbn;
259
# the SQL query to search on isbn
301
# the SQL query to search on isbn
260
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
302
if ($isbn_check) {
303
    $sth_isbn = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE isbn=?");
304
}
261
305
262
my $loghandle;
306
my $loghandle;
263
if ($logfile){
307
if ($logfile) {
264
   $loghandle= IO::File->new($logfile, $writemode) ;
308
    $loghandle= IO::File->new($logfile, $writemode);
265
   print $loghandle "id;operation;status\n";
309
    print $loghandle "id;operation;status\n";
266
}
310
}
267
311
312
my $record_number = 0;
268
my $logger = Koha::Logger->get;
313
my $logger = Koha::Logger->get;
269
my $schema = Koha::Database->schema;
314
my $schema = Koha::Database->schema;
270
$schema->txn_begin;
315
$schema->txn_begin;
316
my $marc_records = [];
271
RECORD: while (  ) {
317
RECORD: while (  ) {
272
    my $record;
318
    my $record;
273
    # get records
319
    $record_number++;
320
    # get record
274
    eval { $record = $batch->next() };
321
    eval { $record = $batch->next() };
275
    if ( $@ ) {
322
    if ($@) {
276
        print "Bad MARC record $i: $@ skipped\n";
323
        print "Bad MARC record $record_number: $@ skipped\n";
277
        # FIXME - because MARC::Batch->next() combines grabbing the next
324
        # FIXME - because MARC::Batch->next() combines grabbing the next
278
        # blob and parsing it into one operation, a correctable condition
325
        # blob and parsing it into one operation, a correctable condition
279
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
326
        # such as a MARC-8 record claiming that it's UTF-8 can't be recovered
Lines 282-560 RECORD: while ( ) { Link Here
282
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
329
        # C4::Charset::MarcToUTF8Record) because it doesn't use MARC::Batch.
283
        next;
330
        next;
284
    }
331
    }
285
    # skip if we get an empty record (that is MARC valid, but will result in AddBiblio failure
332
    if ($record) {
286
    last unless ( $record );
333
        # transcode the record to UTF8 if needed & applicable.
287
    $i++;
334
        if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
288
    if( ($verbose//1)==1 ) { #no dot for verbose==2
335
            my ($guessed_charset, $charset_errors);
289
        print "." . ( $i % 100==0 ? "\n$i" : '' );
336
            ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marc_flavour . (($authorities and $marc_flavour ne "MARC21") ? 'AUTH' : ''));
337
            if ($guessed_charset eq 'failed') {
338
                warn "ERROR: failed to perform character conversion for record $record_number\n";
339
                next RECORD;
340
            }
341
        }
342
        SetUTF8Flag($record);
343
        &$localcust($record) if $localcust;
344
        push @{$marc_records}, $record;
345
    }
346
    else {
347
        last;
290
    }
348
    }
349
}
291
350
292
    # transcode the record to UTF8 if needed & applicable.
351
$record_number = 0;
293
    if ($record->encoding() eq 'MARC-8' and not $skip_marc8_conversion) {
352
my $records_total = @{$marc_records};
294
        # FIXME update condition
353
$schema->txn_begin;
295
        my ($guessed_charset, $charset_errors);
354
RECORD: foreach my $record (@{$marc_records}) {
296
         ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour.(($authorities and $marcFlavour ne "MARC21")?'AUTH':''));
355
    $record_number++;
297
        if ($guessed_charset eq 'failed') {
356
    if (($verbose//1) == 1) { #no dot for verbose==2
298
            warn "ERROR: failed to perform character conversion for record $i\n";
357
        print "." . ($record_number % 100 == 0 ? "\n$record_number" : '');
299
            next RECORD;            
300
        }
301
    }
358
    }
302
    SetUTF8Flag($record);
359
303
    if($marc_mod_template_id > 0) {
360
    if ($marc_mod_template_id > 0) {
304
    print "Modifying MARC\n" if $verbose;
361
        print "Modifying MARC\n" if $verbose;
305
    ModifyRecordWithTemplate( $marc_mod_template_id, $record );
362
        ModifyRecordWithTemplate( $marc_mod_template_id, $record );
306
    }
363
    }
307
    &$localcust($record) if $localcust;
364
308
    my $isbn;
309
    # remove trailing - in isbn (only for biblios, of course)
365
    # remove trailing - in isbn (only for biblios, of course)
310
    if( $biblios ) {
366
    if ($biblios) {
311
        my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
367
        my $tag = $marc_flavour eq 'UNIMARC' ? '010' : '020';
312
        my $field = $record->field($tag);
368
        my $field = $record->field($tag);
313
        $isbn = $field && $field->subfield('a');
369
        my $isbn = $field && $field->subfield('a');
314
        if ( $isbn && $cleanisbn ) {
370
        if ( $isbn && $cleanisbn ) {
315
            $isbn =~ s/-//g;
371
            $isbn =~ s/-//g;
316
            $field->update('a' => $isbn);
372
            $field->update('a' => $isbn);
317
        }
373
        }
318
    }
374
    }
319
    my $id;
320
    # search for duplicates (based on Local-number)
375
    # search for duplicates (based on Local-number)
321
    my $originalid;
376
    my $originalid = GetRecordId($record, $tagid, $subfieldid);
322
    $originalid = GetRecordId( $record, $tagid, $subfieldid );
377
    my $matched_record_id = undef;
323
    if ($match) {
378
    if ($match) {
324
        require C4::Search;
379
        require C4::Search;
325
        my $query = build_query( $match, $record );
380
        my $server = ($authorities ? 'authorityserver' : 'biblioserver');
326
        my $server = ( $authorities ? 'authorityserver' : 'biblioserver' );
381
        my $query = build_query($match, $record);
327
        my ( $error, $results, $totalhits ) = $searcher->simple_search_compat( $query, 0, 3, [$server] );
382
        $logger->debug("Bulkmarcimport: $query");
383
        my ($error, $results, $totalhits) = $searcher->simple_search_compat($query, 0, 3, [$server]);
328
        # changed to warn so able to continue with one broken record
384
        # changed to warn so able to continue with one broken record
329
        if ( defined $error ) {
385
        if (defined $error) {
330
            warn "unable to search the database for duplicates : $error";
386
            warn "unable to search the database for duplicates : $error";
331
            printlog( { id => $id || $originalid || $match, op => "match", status => "ERROR" } ) if ($logfile);
387
            printlog({ id => $originalid , op => "match", status => "ERROR" }) if ($logfile);
332
            next RECORD;
388
            next RECORD;
333
        }
389
        }
334
        if ( $results && scalar(@$results) == 1 ) {
390
        $logger->debug("Bulkmarcimport: $query $server : $totalhits");
335
            my $marcrecord = C4::Search::new_record_from_zebra( $server, $results->[0] );
391
        # sub SimpleSearch could return undefined, but only on error, so
336
            SetUTF8Flag($marcrecord);
392
        # should not really need to safeguard here, but do so anyway
337
            $id = GetRecordId( $marcrecord, $tagid, $subfieldid );
393
        $results //= [];
338
            if ( $authorities && $marcFlavour ) {
394
        if (@{$results} == 1) {
339
                #Skip if authority in database is the same as the on in database
395
            my $matched_record = C4::Search::new_record_from_zebra($server, $results->[0]);
340
                if ( $marcrecord->field('005') && $record->field('005') &&
396
            SetUTF8Flag($matched_record);
341
                     $marcrecord->field('005')->data && $record->field('005')->data &&
397
            $matched_record_id = GetRecordId($matched_record, $tagid, $subfieldid);
342
                     $marcrecord->field('005')->data >= $record->field('005')->data ) {
398
399
            if ($authorities && $marc_flavour) {
400
                #Skip if authority in database is the same or newer than the incoming record
401
                if (RecordRevisionIsGtOrEq($matched_record, $record)) {
343
                    if ($yamlfile) {
402
                    if ($yamlfile) {
344
                        $yamlhash->{$originalid}->{'authid'} = $id;
403
                        $yamlhash->{$originalid} = YAMLFileEntry(
345
404
                            $matched_record,
346
                        # we recover all subfields of the heading authorities
405
                            $matched_record_id,
347
                        my @subfields;
406
                            0
348
                        foreach my $field ( $marcrecord->field("2..") ) {
407
                        );
349
                            push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
350
                        }
351
                        $yamlhash->{$originalid}->{'subfields'} = \@subfields;
352
                        $yamlhash->{$originalid}->{'updated'} = 0;
353
                    }
408
                    }
354
                    next;
409
                    next;
355
                }
410
                }
356
            }
411
            }
357
        } elsif ( $results && scalar(@$results) > 1 ) {
358
            $logger->debug("more than one match for $query");
359
        } else {
360
            $logger->debug("nomatch for $query");
361
        }
412
        }
362
    }
413
        elsif(@{$results} > 1) {
363
    if ($keepids && $originalid) {
414
            $logger->debug("More than one match for: $query");
415
        }
416
        else {
417
            $logger->debug("No match for: $query");
418
        }
419
420
        if ($keepids && $originalid) {
364
            my $storeidfield;
421
            my $storeidfield;
365
            if ( length($keepids) == 3 ) {
422
            if (length($keepids) == 3) {
366
                $storeidfield = MARC::Field->new( $keepids, $originalid );
423
                $storeidfield = MARC::Field->new($keepids, $originalid);
367
            } else {
424
            } else {
368
                $storeidfield = MARC::Field->new( substr( $keepids, 0, 3 ), "", "", substr( $keepids, 3, 1 ), $originalid );
425
                $storeidfield = MARC::Field->new(substr($keepids, 0, 3), "", "", substr($keepids, 3, 1), $originalid);
369
            }
426
            }
370
            $record->insert_fields_ordered($storeidfield);
427
            $record->insert_fields_ordered($storeidfield);
371
            $record->delete_field( $record->field($tagid) );
428
            $record->delete_field($record->field($tagid));
429
        }
372
    }
430
    }
431
373
    foreach my $stringfilter (@$filters) {
432
    foreach my $stringfilter (@$filters) {
374
        if ( length($stringfilter) == 3 ) {
433
        if (length($stringfilter) == 3) {
375
            foreach my $field ( $record->field($stringfilter) ) {
434
            foreach my $field ($record->field($stringfilter)) {
376
                $record->delete_field($field);
435
                $record->delete_field($field);
377
                $logger->debug("removed : ", $field->as_string);
436
                $logger->debug("Removed: ", $field->as_string);
378
            }
437
            }
379
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
438
        } elsif ($stringfilter =~ /([0-9]{3})([a-z0-9])(.*)/) {
380
            my $removetag = $1;
439
            my $removetag = $1;
381
            my $removesubfield = $2;
440
            my $removesubfield = $2;
382
            my $removematch = $3;
441
            my $removematch = $3;
383
            if ( ( $removetag > "010" ) && $removesubfield ) {
442
            if (($removetag > "010") && $removesubfield) {
384
                foreach my $field ( $record->field($removetag) ) {
443
                foreach my $field ($record->field($removetag)) {
385
                    $field->delete_subfield( code => "$removesubfield", match => $removematch );
444
                    $field->delete_subfield(code => "$removesubfield", match => $removematch);
386
                    $logger->debug("Potentially removed : ", $field->subfield($removesubfield));
445
                    $logger->debug("Potentially removed: ", $field->subfield($removesubfield));
387
                }
446
                }
388
            }
447
            }
389
        }
448
        }
390
    }
449
    }
391
    unless ($test_parameter) {
450
    unless ($test_parameter) {
392
        if ($authorities){
451
        if ($authorities) {
393
            my $authtypecode=GuessAuthTypeCode($record, $heading_fields);
452
            my $authtypecode = GuessAuthTypeCode($record, $heading_fields);
394
            my $authid= ($id?$id:GuessAuthId($record));
453
            my $authid = ($matched_record_id? $matched_record_id : GuessAuthId($record));
395
            if ($authid && GetAuthority($authid) && $update ){
454
            if ($authid && GetAuthority($authid) && $update) {
396
            ## Authority has an id and is in database : Replace
455
            ## Authority has an id and is in database : Replace
397
                eval { ( $authid ) = ModAuthority($authid,$record, $authtypecode) };
456
                eval { ( $authid ) = ModAuthority($authid, $record, $authtypecode) };
398
                if ($@){
457
                if ($@) {
399
                    warn "Problem with authority $authid Cannot Modify";
458
                    warn "Problem with authority $authid Cannot Modify";
400
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ERROR"}) if ($logfile);
459
                    printlog({ id => $authid, op => "edit", status => "ERROR" }) if ($logfile);
460
                }
461
                else{
462
                    printlog({ id=> $authid, op=> "edit", status => "ok"}) if ($logfile);
401
                }
463
                }
402
				else{
464
            }
403
					printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
465
            else {
404
				}
405
            }  
406
	        else {
407
            ## True insert in database
466
            ## True insert in database
408
                eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
467
                eval { ( $authid ) = AddAuthority($record, "", $authtypecode) };
409
                if ($@){
468
                if ($@) {
410
                    warn "Problem with authority $authid Cannot Add".$@;
469
                    warn "Problem with authority $originalid Cannot Add ".$@;
411
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
470
                    printlog({ id => $originalid, op => "insert", status => "ERROR" }) if ($logfile);
412
                }
471
                }
413
   				else{
472
                else {
414
					printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
473
                    printlog({ id => $authid, op => "insert", status => "ok" }) if ($logfile);
415
				}
474
                }
416
 	        }
475
417
            if ($yamlfile) {
418
            $yamlhash->{$originalid}->{'authid'} = $authid;
419
            my @subfields;
420
            foreach my $field ( $record->field("2..") ) {
421
                push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
422
            }
476
            }
423
            $yamlhash->{$originalid}->{'subfields'} = \@subfields;
477
            if ($yamlfile) {
424
            $yamlhash->{$originalid}->{'updated'} = 1;
478
                $yamlhash->{$originalid} = YAMLFileEntry(
479
                    $record,
480
                    $authid,
481
                    1 #@FIXME: Really always updated?
482
                );
425
            }
483
            }
426
        }
484
        }
427
        else {
485
        else {
428
            my ( $biblionumber, $biblioitemnumber, $itemnumbers_ref, $errors_ref );
486
            my ($biblioitemnumber, $itemnumbers_ref, $errors_ref, $record_id);
429
            $biblionumber = $id;
430
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
487
            # check for duplicate, based on ISBN (skip it if we already have found a duplicate with match parameter
431
            if (!$biblionumber && $isbn_check && $isbn) {
488
            if (!$matched_record_id && $isbn_check) {
432
    #         warn "search ISBN : $isbn";
489
                my $field = $record->field($marc_flavour eq 'UNIMARC' ? '010' : '020');
490
                my $isbn = $field && $field->subfield('a');
491
                # TODO: Check that isbn has valid valiue, or $field->subfield('a')->data??
433
                $sth_isbn->execute($isbn);
492
                $sth_isbn->execute($isbn);
434
                ($biblionumber,$biblioitemnumber) = $sth_isbn->fetchrow;
493
                ($matched_record_id, $biblioitemnumber) = $sth_isbn->fetchrow;
435
            }
494
            }
436
        	if (defined $idmapfl) {
495
437
			 	if ($sourcetag < "010"){
496
            if (defined $idmapfl && $matched_record_id) {
438
					if ($record->field($sourcetag)){
497
                if ($sourcetag < "010") {
439
					  my $source = $record->field($sourcetag)->data();
498
                    if ($record->field($sourcetag)) {
440
                      printf($idmapfh "%s|%s\n",$source,$biblionumber);
499
                        my $source = $record->field($sourcetag)->data();
441
					}
500
                        printf($idmapfh "%s|%s\n", $source, $matched_record_id);
442
			    } else {
501
                    }
443
					my $source=$record->subfield($sourcetag,$sourcesubfield);
502
                }
444
                    printf($idmapfh "%s|%s\n",$source,$biblionumber);
503
                else {
445
			  }
504
                    my $source = $record->subfield($sourcetag, $sourcesubfield);
446
			}
505
                    printf($idmapfh "%s|%s\n", $source, $matched_record_id);
447
					# create biblio, unless we already have it ( either match or isbn )
506
                }
448
            if ($biblionumber) {
507
            }
508
509
            # create biblio, unless we already have it ( either match or isbn )
510
            if ($matched_record_id) {
511
                # TODO: Implement also for authority records!
449
                eval{
512
                eval{
450
                    $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
513
                    $biblioitemnumber = Koha::Biblios->find( $matched_record_id )->biblioitem->biblioitemnumber;
451
                };
514
                };
452
                if ($update) {
515
                if ($update) {
453
                    eval { ModBiblio( $record, $biblionumber, $framework ) };
516
                    my $success;
517
                    eval { $success = ModBiblio($record, $matched_record_id, GetFrameworkCode($matched_record_id), $modify_biblio_marc_options) };
454
                    if ($@) {
518
                    if ($@) {
455
                        warn "ERROR: Edit biblio $biblionumber failed: $@\n";
519
                        warn "ERROR: Edit biblio $matched_record_id failed: $@\n";
456
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
520
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
457
                        next RECORD;
521
                        next RECORD;
458
                    } else {
459
                        printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ok" } ) if ($logfile);
460
                    }
522
                    }
461
                } else {
523
                    elsif (!$success) {
462
                    printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "warning : already in database" } ) if ($logfile);
524
                        warn "ERROR: Edit biblio $matched_record_id failed for unkown reason";
463
                }
525
                        printlog( { id => $matched_record_id, op => "update", status => "ERROR" } ) if ($logfile);
464
            } else {
465
                if ($insert) {
466
                    eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
467
                    if ($@) {
468
                        warn "ERROR: Adding biblio $biblionumber failed: $@\n";
469
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
470
                        next RECORD;
526
                        next RECORD;
471
                    } else {
472
                        printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
473
                    }
527
                    }
474
                } else {
528
                    else {
475
                    warn "WARNING: Updating record ".($id||$originalid)." failed";
529
                        $record_id = $matched_record_id;
476
                    printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
530
                        printlog( { id => $record_id, op => "update", status => "ok" } ) if ($logfile);
531
                    }
532
                }
533
                else {
534
                    printlog( { id => $matched_record_id, op => "update", status => "warning : already in database and option -update not enabled, skipping..." } ) if ($logfile);
535
                }
536
            }
537
            elsif ($insert) {
538
                eval { ($record_id, $biblioitemnumber) = AddBiblio($record, $framework, { defer_marc_save => 1 }) };
539
                if ($@) {
540
                    warn "ERROR: Adding biblio $record_id failed: $@\n";
541
                    printlog( { id => $record_id, op => "insert", status => "ERROR" } ) if ($logfile);
477
                    next RECORD;
542
                    next RECORD;
478
                }
543
                }
544
                else {
545
                    printlog( { id => $record_id, op => "insert", status => "ok" } ) if ($logfile);
546
                }
479
            }
547
            }
480
            eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
548
            else {
481
            my $error_adding = $@;
549
                warn "WARNING: Updating record ".($originalid)." failed";
482
            # Work on a clone so that if there are real errors, we can maybe
550
                printlog( { id => $originalid, op => "insert", status => "warning : not in database and option -insert not enabled, skipping..." } ) if ($logfile);
483
            # fix them up later.
484
			my $clone_record = $record->clone();
485
            C4::Biblio::_strip_item_fields($clone_record, '');
486
            # This sets the marc fields if there was an error, and also calls
487
            # defer_marc_save.
488
            ModBiblioMarc( $clone_record, $biblionumber );
489
            if ( $error_adding ) {
490
                warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
491
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
492
                # if we failed because of an exception, assume that 
493
                # the MARC columns in biblioitems were not set.
494
                next RECORD;
551
                next RECORD;
495
            }
552
            }
496
 			else{
553
            my $record_has_added_items = 0;
497
				printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
554
            if ($record_id) {
498
			}
555
                $yamlhash->{$originalid} = $record_id if $yamlfile;
499
            if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
556
                # TODO: Add option for skipping items?
500
                # Find the record called 'barcode'
557
                eval { ($itemnumbers_ref, $errors_ref) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, $framework); };
501
                my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
558
                $record_has_added_items = @{$itemnumbers_ref};
502
                # Now remove any items that didn't have a duplicate_barcode error,
559
                my $error_adding = $@;
503
                # erase the barcodes on items that did, and re-add those items.
560
                # Work on a clone so that if there are real errors, we can maybe
504
                my %dupes;
561
                # fix them up later.
505
                foreach my $i (0 .. $#{$errors_ref}) {
562
                my $clone_record = $record->clone();
506
                    my $ref = $errors_ref->[$i];
563
                C4::Biblio::_strip_item_fields($clone_record, $framework);
507
                    if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
564
                # This sets the marc fields if there was an error, and also calls
508
                        $dupes{$ref->{item_sequence}} = 1;
565
                # defer_marc_save.
509
                        # Delete the error message because we're going to
566
                ModBiblioMarc($clone_record, $record_id, $modify_biblio_marc_options);
510
                        # retry this one.
567
                if ($error_adding) {
511
                        delete $errors_ref->[$i];
568
                    warn "ERROR: Adding items to bib $record_id failed: $error_adding";
512
                    }
569
                    printlog({ id => $record_id, op => "insert items", status => "ERROR"}) if ($logfile);
513
                }
514
                my $seq = 0;
515
                foreach my $field ($record->field($tag)) {
516
                    $seq++;
517
                    if ($dupes{$seq}) {
518
                        # Here we remove the barcode
519
                        $field->delete_subfield(code => $sub);
520
                    } else {
521
                        # otherwise we delete the field because we don't want
522
                        # two of them
523
                        $record->delete_fields($field);
524
                    }
525
                }
526
                # Now re-add the record as before, adding errors to the prev list
527
                my $more_errors;
528
                eval { ( $itemnumbers_ref, $more_errors ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
529
                if ( $@ ) {
530
                    warn "ERROR: Adding items to bib $biblionumber failed: $@\n";
531
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
532
                    # if we failed because of an exception, assume that
570
                    # if we failed because of an exception, assume that
533
                    # the MARC columns in biblioitems were not set.
571
                    # the MARC columns in biblioitems were not set.
534
                    ModBiblioMarc( $record, $biblionumber );
535
                    next RECORD;
572
                    next RECORD;
536
                } else {
537
                    printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
538
                }
573
                }
539
                push @$errors_ref, @{ $more_errors };
574
                else {
540
            }
575
                    printlog({ id => $record_id, op => "insert items", status => "ok" }) if ($logfile);
541
            if ($#{ $errors_ref } > -1) {
576
                }
542
                report_item_errors($biblionumber, $errors_ref);
577
                if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
578
                    # Find the record called 'barcode'
579
                    my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode');
580
                    # Now remove any items that didn't have a duplicate_barcode error,
581
                    # erase the barcodes on items that did, and re-add those items.
582
                    my %dupes;
583
                    # FIXME: This could cause array out of bounds because shifting down rest of items on array item delete?
584
                    foreach my $i (0 .. $#{$errors_ref}) {
585
                        my $ref = $errors_ref->[$i];
586
                        if ($ref && ($ref->{error_code} eq 'duplicate_barcode')) {
587
                            $dupes{$ref->{item_sequence}} = 1;
588
                            # Delete the error message because we're going to
589
                            # retry this one.
590
                            delete $errors_ref->[$i];
591
                        }
592
                    }
593
                    my $seq = 0;
594
                    foreach my $field ($record->field($tag)) {
595
                        $seq++;
596
                        if ($dupes{$seq}) {
597
                            # Here we remove the barcode
598
                            $field->delete_subfield(code => $sub);
599
                        }
600
                        else {
601
                            # otherwise we delete the field because we don't want
602
                            # two of them
603
                            $record->delete_fields($field);
604
                        }
605
                    }
606
                    # Now re-add the record as before, adding errors to the prev list
607
                    my $more_errors;
608
                    eval { ($itemnumbers_ref, $more_errors) = AddItemBatchFromMarc($record, $record_id, $biblioitemnumber, ''); };
609
                    $record_has_added_items ||= @{$itemnumbers_ref};
610
                    if ($@) {
611
                        warn "ERROR: Adding items to bib $record_id failed: $@\n";
612
                        printlog({ id => $record_id, op => "insert items", status => "ERROR" }) if ($logfile);
613
                        # if we failed because of an exception, assume that
614
                        # the MARC columns in biblioitems were not set.
615
616
                        # @FIXME: Why do we save here without stripping items? Besides,
617
                        # save with stripped items has already been performed
618
                        ModBiblioMarc($record, $record_id, $modify_biblio_marc_options);
619
                        next RECORD;
620
                    }
621
                    else {
622
                        printlog({ id => $record_id, op => "insert", status => "ok" }) if ($logfile);
623
                    }
624
                    push @$errors_ref, @{$more_errors};
625
                }
626
                if (@{$errors_ref}) {
627
                    report_item_errors($record_id, $errors_ref);
628
                }
629
                C4::Biblio::_strip_item_fields($record, $framework);
630
                if ($record_has_added_items || $matched_record_id) {
631
                    # Replace with record from GetMarcBiblio with "$embeditems = 1"
632
                    $record = GetMarcBiblio({biblionumber => $record_id, embed_items => 1});
633
                }
634
                push @search_engine_record_ids, $record_id;
635
                push @search_engine_records, $record;
543
            }
636
            }
544
            $yamlhash->{$originalid} = $biblionumber if ($yamlfile);
545
        }
637
        }
546
        if ( 0 == $i % $commitnum ) {
638
        if ($record_number % $commitnum == 0 || $record_number == $records_total) {
547
            $schema->txn_commit;
639
            $schema->txn_commit;
548
            $schema->txn_begin;
640
            $schema->txn_begin;
641
            if ($indexer) {
642
                $indexer->update_index(\@search_engine_record_ids, \@search_engine_records);
643
                @search_engine_record_ids = ();
644
                @search_engine_records = ();
645
            }
549
        }
646
        }
550
    }
647
    }
551
    print $record->as_formatted()."\n" if ($verbose//0)==2;
648
    print $record->as_formatted() . "\n" if ($verbose//0) == 2;
552
    last if $i == $number;
649
    last if $record_number == $number;
553
}
650
}
554
$schema->txn_commit;
651
$schema->txn_commit;
555
652
556
if ($fk_off) {
653
if ($fk_off) {
557
	$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
654
    $dbh->do("SET FOREIGN_KEY_CHECKS = 1");
558
}
655
}
559
656
560
# Restore CataloguingLog and AuthoritiesLog
657
# Restore CataloguingLog and AuthoritiesLog
Lines 562-572 delete $ENV{OVERRIDE_SYSPREF_CataloguingLog}; Link Here
562
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
659
delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
563
660
564
my $timeneeded = gettimeofday - $starttime;
661
my $timeneeded = gettimeofday - $starttime;
565
print "\n$i MARC records done in $timeneeded seconds\n";
662
print "\n$record_number MARC records done in $timeneeded seconds\n";
566
if ($logfile){
663
if ($logfile) {
567
  print $loghandle "file : $input_marc_file\n";
664
    print $loghandle "file : $input_marc_file\n";
568
  print $loghandle "$i MARC records done in $timeneeded seconds\n";
665
    print $loghandle "$record_number MARC records done in $timeneeded seconds\n";
569
  $loghandle->close;
666
    $loghandle->close;
570
}
667
}
571
if ($yamlfile) {
668
if ($yamlfile) {
572
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
669
    open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
Lines 574-620 if ($yamlfile) { Link Here
574
}
671
}
575
exit 0;
672
exit 0;
576
673
577
sub GetRecordId{
674
sub YAMLFileEntry {
578
	my $marcrecord=shift;
675
    my ($record, $record_id, $updated) = @_;
579
	my $tag=shift;
676
580
	my $subfield=shift;
677
    my $entry = {
581
	my $id;
678
        authid => $record_id
582
	if ($tag lt "010"){
679
    };
583
		return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
680
584
	} 
681
    # we recover all subfields of the heading authorities
585
	elsif ($subfield){
682
    my @subfields;
586
		if ($marcrecord->field($tag)){
683
    foreach my $field ($record->field("2..")) {
587
			return $marcrecord->subfield($tag,$subfield);
684
        push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
588
		}
685
    }
589
	}
686
    $entry->{'subfields'} = \@subfields;
590
	return $id;
687
    $entry->{'updated'} = $updated;
688
689
    return $entry;
591
}
690
}
592
sub build_query {
691
593
	my $match = shift;
692
sub RecordRevisionIsGtOrEq {
594
	my $record=shift;
693
    my ($record_a, $record_b) = @_;
595
        my @searchstrings;
694
    return $record_a->field('005') && $record_b->field('005') &&
596
	foreach my $matchingpoint (@$match){
695
    $record_a->field('005')->data && $record_b->field('005')->data &&
597
	  my $string = build_simplequery($matchingpoint,$record);
696
    $record_a->field('005')->data >= $record_b->field('005')->data;
598
	  push @searchstrings,$string if (length($string)>0);
697
}
698
699
sub GetRecordId {
700
    my $marcrecord = shift;
701
    my $tag = shift;
702
    my $subfield = shift;
703
    if ($tag lt "010") {
704
        return $marcrecord->field($tag)->data() if $marcrecord->field($tag);
705
    }
706
    elsif ($subfield) {
707
        if ($marcrecord->field($tag)) {
708
            return $marcrecord->subfield($tag, $subfield);
599
        }
709
        }
710
    }
711
    return undef;
712
}
713
sub build_query {
714
    my ($match, $record) = @_;
715
    my @searchstrings;
716
717
    foreach my $matchpoint (@$match) {
718
        my $query = build_simplequery($matchpoint, $record);
719
        push (@searchstrings, $query) if $query;
720
    }
600
    my $op = 'and';
721
    my $op = 'and';
601
    return join(" $op ",@searchstrings);
722
    return join(" $op ", @searchstrings);
602
}
723
}
603
sub build_simplequery {
724
sub build_simplequery {
604
	my $element=shift;
725
    my ($matchpoint, $record) = @_;
605
	my $record=shift;
726
606
    my @searchstrings;
727
    my @searchstrings;
607
    my ($index,$recorddata)=split /,/,$element;
728
    my ($index, $record_data) = split (/,/, $matchpoint);
608
    if ($recorddata=~/(\d{3})(.*)/) {
729
    if ($record_data =~ /(\d{3})(.*)/) {
609
        my ($tag,$subfields) =($1,$2);
730
        my ($tag, $subfields) = ($1, $2);
610
        foreach my $field ($record->field($tag)){
731
        foreach my $field ($record->field($tag)) {
611
		  if (length($field->as_string("$subfields"))>0){
732
            if (length($field->as_string("$subfields")) > 0) {
612
              push @searchstrings,"$index:\"".$field->as_string("$subfields")."\"";
733
                push (@searchstrings, "$index:\"" . $field->as_string("$subfields") . "\"");
613
		  }
734
            }
614
        }
735
        }
615
    }
736
    }
737
    else {
738
        print "Invalid matchpoint format, invalid marc-field: $matchpoint\n";
739
    }
616
    my $op = 'and';
740
    my $op = 'and';
617
    return join(" $op ",@searchstrings);
741
    return join(" $op ", @searchstrings);
618
}
742
}
619
sub report_item_errors {
743
sub report_item_errors {
620
    my $biblionumber = shift;
744
    my $biblionumber = shift;
Lines 629-648 sub report_item_errors { Link Here
629
        print $msg, "\n";
753
        print $msg, "\n";
630
    }
754
    }
631
}
755
}
632
sub printlog{
756
sub printlog {
633
	my $logelements=shift;
757
    my $logelements = shift;
634
    print $loghandle join( ";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>} ), "\n";
758
    print $loghandle join(";", map { defined $_ ? $_ : "" } @$logelements{qw<id op status>}), "\n";
635
}
759
}
636
sub get_heading_fields{
760
sub get_heading_fields {
637
    my $headingfields;
761
    my $headingfields;
638
    if ($authtypes){
762
    if ($authtypes) {
639
        $headingfields = YAML::XS::LoadFile($authtypes);
763
        $headingfields = YAML::XS::LoadFile($authtypes);
640
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
764
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
641
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
765
        $logger->debug(Encode::decode_utf8(YAML::XS::Dump($headingfields)));
642
    }
766
    }
643
    unless ($headingfields){
767
    unless ($headingfields) {
644
        $headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
768
        $headingfields = $dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
645
        $headingfields={C4::Context->preference('marcflavour')=>$headingfields};
769
        $headingfields = { C4::Context->preference('marcflavour') => $headingfields };
646
    }
770
    }
647
    return $headingfields;
771
    return $headingfields;
648
}
772
}
(-)a/t/db_dependent/Biblio.t (-2 / +1 lines)
Lines 738-744 subtest 'ModBiblio called from linker test' => sub { Link Here
738
    C4::Biblio::ModBiblio($record,$biblionumber,'');
738
    C4::Biblio::ModBiblio($record,$biblionumber,'');
739
    is($called,1,"We called to link bibs because not from linker");
739
    is($called,1,"We called to link bibs because not from linker");
740
    $called = 0;
740
    $called = 0;
741
    C4::Biblio::ModBiblio($record,$biblionumber,'',1);
741
    C4::Biblio::ModBiblio($record,$biblionumber,'',{ disable_autolink => 1 });
742
    is($called,0,"We didn't call to link bibs because from linker");
742
    is($called,0,"We didn't call to link bibs because from linker");
743
};
743
};
744
744
745
- 

Return to bug 29440