View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-87 / +212 lines)
Lines 73-78 sub new { Link Here
73
    return $self;
73
    return $self;
74
}
74
}
75
75
76
=head2 get_elasticsearch
77
78
    my $elasticsearch_client = $self->get_elasticsearch();
79
80
Returns a C<Search::Elasticsearch> client. The client is cached on a C<Koha::SearchEngine::ElasticSearch>
81
instance level and will be reused if method is called multiple times.
82
83
=cut
84
76
sub get_elasticsearch {
85
sub get_elasticsearch {
77
    my $self = shift @_;
86
    my $self = shift @_;
78
    unless (defined $self->{elasticsearch}) {
87
    unless (defined $self->{elasticsearch}) {
Lines 145-152 sub get_elasticsearch_params { Link Here
145
154
146
    my $settings = $self->get_elasticsearch_settings();
155
    my $settings = $self->get_elasticsearch_settings();
147
156
148
This provides the settings provided to elasticsearch when an index is created.
157
This provides the settings provided to Elasticsearch when an index is created.
149
These can do things like define tokenisation methods.
158
These can do things like define tokenization methods.
150
159
151
A hashref containing the settings is returned.
160
A hashref containing the settings is returned.
152
161
Lines 170-176 sub get_elasticsearch_settings { Link Here
170
179
171
    my $mappings = $self->get_elasticsearch_mappings();
180
    my $mappings = $self->get_elasticsearch_mappings();
172
181
173
This provides the mappings that get passed to elasticsearch when an index is
182
This provides the mappings that get passed to Elasticsearch when an index is
174
created.
183
created.
175
184
176
=cut
185
=cut
Lines 232-238 sub get_elasticsearch_mappings { Link Here
232
241
233
=head2 _get_elasticsearch_mapping
242
=head2 _get_elasticsearch_mapping
234
243
235
Get the ES mappings for the given purpose and data type
244
Get the Elasticsearch mappings for the given purpose and data type.
236
245
237
$mapping = _get_elasticsearch_mapping('search', 'text');
246
$mapping = _get_elasticsearch_mapping('search', 'text');
238
247
Lines 301-351 sub sort_fields { Link Here
301
    return $self->_sort_fields_accessor();
310
    return $self->_sort_fields_accessor();
302
}
311
}
303
312
313
=head2 _process_mappings($mappings, $data, $record_document)
314
315
Process all C<$mappings> targets operating on a specific MARC field C<$data> applied to C<$record_document>
316
Since we group all mappings by MARC field targets C<$mappings> will contain all targets for C<$data>
317
and thus we need to fetch the MARC field only once.
318
319
=over 4
320
321
=item C<$mappings>
322
323
Arrayref of mappings containing arrayrefs on the format [C<$taget>, C<$options>] where
324
C<$target> is the name of the target field and C<$options> is a hashref containing processing
325
directives for this particular mapping.
326
327
=item C<$data>
328
329
The source data from a MARC record field.
330
331
=item C<$record_document>
332
333
Hashref representing the  Elasticsearch document on which mappings should be applied.
334
335
=back
336
337
=cut
338
339
sub _process_mappings {
340
    my ($_self, $mappings, $data, $record_document) = @_;
341
    foreach my $mapping (@{$mappings}) {
342
        my ($target, $options) = @{$mapping};
343
        # Copy (scalar) data since can have multiple targets
344
        # with differing options for (possibly) mutating data
345
        # so need a different copy for each
346
        my $_data = $data;
347
        $record_document->{$target} //= [];
348
        if (defined $options->{substr}) {
349
            my ($start, $length) = @{$options->{substr}};
350
            $_data = length($data) > $start ? substr $data, $start, $length : '';
351
        }
352
        if (defined $options->{value_callbacks}) {
353
            $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
354
        }
355
        if (defined $options->{property}) {
356
            $_data = {
357
                $options->{property} => $_data
358
            }
359
        }
360
        push @{$record_document->{$target}}, $_data;
361
    }
362
}
363
364
=head2 marc_records_to_documents($marc_records)
365
366
    my @record_documents = $self->marc_records_to_documents($marc_records);
367
368
Using mappings stored in database convert C<$marc_records> to Elasticsearch documents.
369
370
Returns array of hash references, representing Elasticsearch documents,
371
acceptable as body payload in C<Search::Elasticsearch> requests.
372
373
=over 4
374
375
=item C<$marc_documents>
376
377
Reference to array of C<MARC::Record> objects to be converted to Elasticsearch documents.
378
379
=back
380
381
=cut
382
304
sub marc_records_to_documents {
383
sub marc_records_to_documents {
305
    my ($self, $records) = @_;
384
    my ($self, $records) = @_;
306
    my $rules = $self->get_marc_mapping_rules();
385
    my $rules = $self->_get_marc_mapping_rules();
307
    my $control_fields_rules = $rules->{control_fields};
386
    my $control_fields_rules = $rules->{control_fields};
308
    my $data_fields_rules = $rules->{data_fields};
387
    my $data_fields_rules = $rules->{data_fields};
309
    my $marcflavour = lc C4::Context->preference('marcflavour');
388
    my $marcflavour = lc C4::Context->preference('marcflavour');
310
    my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
311
389
312
    my @record_documents;
390
    my @record_documents;
313
391
314
    sub _process_mappings {
315
        my ($mappings, $data, $record_document) = @_;
316
        foreach my $mapping (@{$mappings}) {
317
            my ($target, $options) = @{$mapping};
318
            # Copy (scalar) data since can have multiple targets
319
            # with differing options for (possibly) mutating data
320
            # so need a different copy for each
321
            my $_data = $data;
322
            $record_document->{$target} //= [];
323
            if (defined $options->{substr}) {
324
                my ($start, $length) = @{$options->{substr}};
325
                $_data = length($data) > $start ? substr $data, $start, $length : '';
326
            }
327
            if (defined $options->{value_callbacks}) {
328
                $_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
329
            }
330
            if (defined $options->{property}) {
331
                $_data = {
332
                    $options->{property} => $_data
333
                }
334
            }
335
            push @{$record_document->{$target}}, $_data;
336
        }
337
    }
338
    foreach my $record (@{$records}) {
392
    foreach my $record (@{$records}) {
339
        my $record_document = {};
393
        my $record_document = {};
340
        my $mappings = $rules->{leader};
394
        my $mappings = $rules->{leader};
341
        if ($mappings) {
395
        if ($mappings) {
342
            _process_mappings($mappings, $record->leader(), $record_document);
396
            $self->_process_mappings($mappings, $record->leader(), $record_document);
343
        }
397
        }
344
        foreach my $field ($record->fields()) {
398
        foreach my $field ($record->fields()) {
345
            if($field->is_control_field()) {
399
            if($field->is_control_field()) {
346
                my $mappings = $control_fields_rules->{$field->tag()};
400
                my $mappings = $control_fields_rules->{$field->tag()};
347
                if ($mappings) {
401
                if ($mappings) {
348
                    _process_mappings($mappings, $field->data(), $record_document);
402
                    $self->_process_mappings($mappings, $field->data(), $record_document);
349
                }
403
                }
350
            }
404
            }
351
            else {
405
            else {
Lines 361-367 sub marc_records_to_documents { Link Here
361
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
415
                            $mappings = [@{$mappings}, @{$wildcard_mappings}];
362
                        }
416
                        }
363
                        if (@{$mappings}) {
417
                        if (@{$mappings}) {
364
                            _process_mappings($mappings, $data, $record_document);
418
                            $self->_process_mappings($mappings, $data, $record_document);
365
                        }
419
                        }
366
                    }
420
                    }
367
421
Lines 377-383 sub marc_records_to_documents { Link Here
377
                                )
431
                                )
378
                            );
432
                            );
379
                            if ($data) {
433
                            if ($data) {
380
                                _process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document);
434
                                $self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document);
381
                            }
435
                            }
382
                        }
436
                        }
383
                    }
437
                    }
Lines 426-488 sub marc_records_to_documents { Link Here
426
    return \@record_documents;
480
    return \@record_documents;
427
}
481
}
428
482
429
# Provides the rules for marc to Elasticsearch JSON document conversion.
483
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
430
sub get_marc_mapping_rules {
431
    my ($self) = @_;
432
484
433
    my $marcflavour = lc C4::Context->preference('marcflavour');
485
Get mappings, an internal data structure later used by L<_process_mappings($mappings, $data, $record_document)>
434
    my @rules;
486
to process MARC target data, for a MARC mapping.
487
488
The returned C<$mappings> is to to be confused  with mappings provided by C<_foreach_mapping>, rather this
489
sub accepts properties from a mapping as provided by C<_foreach_mapping> and expands it to this internal
490
data stucture. In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> is then
491
applied to each MARC target (leader, control field data, subfield or joined subfields) and
492
integrated into the mapping rules data structure used in C<marc_records_to_documents> to
493
transform MARC records into Elasticsearch documents.
494
495
=over 4
435
496
436
    sub _field_mappings {
497
=item C<$facet>
437
        my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
498
438
        my %mapping_defaults = ();
499
Boolean indicating whether to create a facet field for this mapping.
439
        my @mappings;
500
440
501
=item C<$suggestible>
441
        my $substr_args = undef;
502
442
        if ($range) {
503
Boolean indicating whether to create a suggestion field for this mapping.
443
            # TODO: use value_callback instead?
504
444
            my ($start, $end) = map(int, split /-/, $range, 2);
505
=item C<$sort>
445
            $substr_args = [$start];
506
446
            push @{$substr_args}, (defined $end ? $end - $start + 1 : 1);
507
Boolean indicating whether to create a sort field for this mapping.
508
509
=item C<$target_name>
510
511
Elasticsearch document target field name.
512
513
=item C<$target_type>
514
515
Elasticsearch document target field type.
516
517
=item C<$range>
518
519
An optinal range as a string on the format "<START>-<END>" or "<START>",
520
where "<START>" and "<END>" are integers specifying a range that will be used
521
for extracting a substing from MARC data as Elasticsearch field target value.
522
523
The first character position is "1", and the range is inclusive,
524
so "1-3" means the first three characters of MARC data.
525
526
If only "<START>" is provided only one character as position "<START>" will
527
be extracted.
528
529
=back
530
531
=cut
532
533
sub _field_mappings {
534
    my ($_self, $facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
535
    my %mapping_defaults = ();
536
    my @mappings;
537
538
    my $substr_args = undef;
539
    if ($range) {
540
        # TODO: use value_callback instead?
541
        my ($start, $end) = map(int, split /-/, $range, 2);
542
        $substr_args = [$start];
543
        push @{$substr_args}, (defined $end ? $end - $start + 1 : 1);
544
    }
545
    my $default_options = {};
546
    if ($substr_args) {
547
        $default_options->{substr} = $substr_args;
548
    }
549
550
    # TODO: Should probably have per type value callback/hook
551
    # but hard code for now
552
    if ($target_type eq 'boolean') {
553
        $default_options->{value_callbacks} //= [];
554
        push @{$default_options->{value_callbacks}}, sub {
555
            my ($value) = @_;
556
            # Trim whitespace at both ends
557
            $value =~ s/^\s+|\s+$//g;
558
            return $value ? 'true' : 'false';
559
        };
560
    }
561
562
    my $mapping = [$target_name, $default_options];
563
    push @mappings, $mapping;
564
565
    my @suffixes = ();
566
    push @suffixes, 'facet' if $facet;
567
    push @suffixes, 'suggestion' if $suggestible;
568
    push @suffixes, 'sort' if !defined $sort || $sort;
569
570
    foreach my $suffix (@suffixes) {
571
        my $mapping = ["${target_name}__$suffix"];
572
        # TODO: Hack, fix later in less hideous manner
573
        if ($suffix eq 'suggestion') {
574
            push @{$mapping}, {%{$default_options}, property => 'input'};
447
        }
575
        }
448
        my $default_options = {};
576
        else {
449
        if ($substr_args) {
577
            push @{$mapping}, $default_options;
450
            $default_options->{substr} = $substr_args;
451
        }
578
        }
579
        push @mappings, $mapping;
580
    }
581
    return @mappings;
582
};
452
583
453
        # TODO: Should probably have per type value callback/hook
584
=head2 _get_marc_mapping_rules
454
        # but hard code for now
455
        if ($target_type eq 'boolean') {
456
            $default_options->{value_callbacks} //= [];
457
            push @{$default_options->{value_callbacks}}, sub {
458
                my ($value) = @_;
459
                # Trim whitespace at both ends
460
                $value =~ s/^\s+|\s+$//g;
461
                return $value ? 'true' : 'false';
462
            };
463
        }
464
585
465
        my $mapping = [$target_name, $default_options];
586
    my $mapping_rules = $self->_get_marc_mapping_rules()
466
        push @mappings, $mapping;
467
587
468
        my @suffixes = ();
588
Generates rules from mappings stored in database for MARC records to Elasticsearch JSON document conversion.
469
        push @suffixes, 'facet' if $facet;
589
470
        push @suffixes, 'suggestion' if $suggestible;
590
Since field retrieval is slow in C<MARC::Records> (all fields are itereted through for
471
        push @suffixes, 'sort' if !defined $sort || $sort;
591
each call to C<MARC::Record>->field) we create an optimized structure of mapping
592
rules keyed by MARC field tags holding all the mapping rules for that particular tag.
593
594
We can then iterate through all MARC fields for each record and apply all relevant
595
rules once per fields instead of retreiving fields multiple times for each mapping rule
596
wich is terribly slow.
597
598
=cut
599
600
# TODO: This structure can be used for processing multiple MARC::Records so is currently
601
# rebuilt for each batch. Since it is cacheable it could also be stored in an in
602
# memory cache which it is currently not. The performance gain of caching
603
# would probably be marginal, but to do this could be a further improvement.
604
605
sub _get_marc_mapping_rules {
606
    my ($self) = @_;
607
608
    my $marcflavour = lc C4::Context->preference('marcflavour');
609
    my @rules;
472
610
473
        foreach my $suffix (@suffixes) {
474
            my $mapping = ["${target_name}__$suffix"];
475
            # Hack, fix later in less hideous manner
476
            if ($suffix eq 'suggestion') {
477
                push @{$mapping}, {%{$default_options}, property => 'input'};
478
            }
479
            else {
480
                push @{$mapping}, $default_options;
481
            }
482
            push @mappings, $mapping;
483
        }
484
        return @mappings;
485
    };
486
    my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
611
    my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
487
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
612
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
488
    my $rules = {
613
    my $rules = {
Lines 494-500 sub get_marc_mapping_rules { Link Here
494
    };
619
    };
495
620
496
    $self->_foreach_mapping(sub {
621
    $self->_foreach_mapping(sub {
497
        my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
622
        my ($name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field) = @_;
498
        return if $marc_type ne $marcflavour;
623
        return if $marc_type ne $marcflavour;
499
624
500
        if ($type eq 'sum') {
625
        if ($type eq 'sum') {
Lines 550-556 sub get_marc_mapping_rules { Link Here
550
            }
675
            }
551
676
552
            my $range = defined $3 ? $3 : undef;
677
            my $range = defined $3 ? $3 : undef;
553
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
678
            my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range);
554
679
555
            if ($field_tag < 10) {
680
            if ($field_tag < 10) {
556
                $rules->{control_fields}->{$field_tag} //= [];
681
                $rules->{control_fields}->{$field_tag} //= [];
Lines 570-580 sub get_marc_mapping_rules { Link Here
570
        }
695
        }
571
        elsif ($marc_field =~ $leader_regexp) {
696
        elsif ($marc_field =~ $leader_regexp) {
572
            my $range = defined $1 ? $1 : undef;
697
            my $range = defined $1 ? $1 : undef;
573
            my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
698
            my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range);
574
            push @{$rules->{leader}}, @mappings;
699
            push @{$rules->{leader}}, @mappings;
575
        }
700
        }
576
        else {
701
        else {
577
            die("Invalid marc field: $marc_field");
702
            die("Invalid MARC field: $marc_field");
578
        }
703
        }
579
    });
704
    });
580
    return $rules;
705
    return $rules;
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-49 / +161 lines)
Lines 44-88 Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i Link Here
44
    $indexer->drop_index();
44
    $indexer->drop_index();
45
    $indexer->update_index(\@biblionumbers, \@records);
45
    $indexer->update_index(\@biblionumbers, \@records);
46
46
47
=head1 FUNCTIONS
48
47
49
=head2 $indexer->update_index($biblionums, $records);
48
=head1 CONSTANTS
50
49
51
C<$biblionums> is an arrayref containing the biblionumbers for the records.
50
=over 4
52
51
53
C<$records> is an arrayref containing the L<MARC::Record>s themselves.
52
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_OK>
54
53
55
The values in the arrays must match up, and the 999$c value in the MARC record
54
Represents an index state where index is created and in a working state.
56
will be rewritten using the values in C<$biblionums> to ensure they are correct.
57
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
58
999$c is correct on your own then.
59
55
60
Note that this will modify the original record if C<$biblionums> is supplied.
56
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_REINDEX_REQUIRED>
61
If that's a problem, clone them first.
57
58
Not currently used, but could be useful later, for example if can detect when new field or mapping added.
59
60
=item C<Koha::SearchEngine::Elasticsearch::Indexer::INDEX_STATUS_RECREATE_REQUIRED>
61
62
Representings an index state where index needs to be recreated and is not in a working state.
63
64
=back
62
65
63
=cut
66
=cut
64
67
65
use constant {
68
use constant {
66
    INDEX_STATUS_OK => 0,
69
    INDEX_STATUS_OK => 0,
67
    INDEX_STATUS_REINDEX_REQUIRED => 1, # Not currently used, but could be useful later, for example if can detect when new field or mapping added
70
    INDEX_STATUS_REINDEX_REQUIRED => 1,
68
    INDEX_STATUS_RECREATE_REQUIRED => 2,
71
    INDEX_STATUS_RECREATE_REQUIRED => 2,
69
};
72
};
70
73
74
=head1 FUNCTIONS
75
76
=head2 update_index($biblionums, $records)
77
78
    try {
79
        $self->update_index($biblionums, $records);
80
    } catch {
81
        die("Something whent wrong trying to update index:" .  $_[0]);
82
    }
83
84
Converts C<MARC::Records> C<$records> to Elasticsearch documents and performs
85
an update request for these records on the Elasticsearch index.
86
87
The values in the arrays must match up, and the 999$c value in the MARC record
88
will be rewritten using the values in C<$biblionums> to ensure they are correct.
89
If C<$biblionums> is C<undef>, this won't happen, so in that case you should make
90
sure that 999$c is correct.
91
92
Note that this will modify the original record if C<$biblionums> is supplied.
93
If that's a problem, clone them first.
94
95
=over 4
96
97
=item C<$biblionums>
98
99
Arrayref of biblio numbers for the C<$records>, the order must be the same as
100
and match up with C<$records>.
101
102
=item C<$records>
103
104
Arrayref of C<MARC::Record>s.
105
106
=back
107
108
=cut
109
71
sub update_index {
110
sub update_index {
72
    my ($self, $biblionums, $records) = @_;
111
    my ($self, $biblionums, $records) = @_;
73
112
74
    # TODO should have a separate path for dealing with a large number
75
    # of records at once where we use the bulk update functions in ES.
76
    if ($biblionums) {
113
    if ($biblionums) {
77
        $self->_sanitise_records($biblionums, $records);
114
        $self->_sanitise_records($biblionums, $records);
78
    }
115
    }
79
116
80
    $self->bulk_index($records);
81
    return 1;
82
}
83
84
sub bulk_index {
85
    my ($self, $records) = @_;
86
    my $conf = $self->get_elasticsearch_params();
117
    my $conf = $self->get_elasticsearch_params();
87
    my $elasticsearch = $self->get_elasticsearch();
118
    my $elasticsearch = $self->get_elasticsearch();
88
    my $documents = $self->marc_records_to_documents($records);
119
    my $documents = $self->marc_records_to_documents($records);
Lines 108-134 sub bulk_index { Link Here
108
    return 1;
139
    return 1;
109
}
140
}
110
141
111
sub index_status_ok {
142
=head2 set_index_status_ok
112
    my ($self, $set) = @_;
143
113
    return defined $set ?
144
Convenience method for setting index status to C<INDEX_STATUS_OK>.
114
        $self->index_status(INDEX_STATUS_OK) :
145
115
        $self->index_status == INDEX_STATUS_OK;
146
=cut
147
148
sub set_index_status_ok {
149
    my ($self) = @_;
150
    $self->index_status(INDEX_STATUS_OK);
116
}
151
}
117
152
118
sub index_status_reindex_required {
153
=head2 is_index_status_ok
119
    my ($self, $set) = @_;
154
120
    return defined $set ?
155
Convenience method for checking if index status is C<INDEX_STATUS_OK>.
121
        $self->index_status(INDEX_STATUS_REINDEX_REQUIRED) :
156
122
        $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
157
=cut
158
159
sub is_index_status_ok {
160
    my ($self) = @_;
161
    return $self->index_status == INDEX_STATUS_OK;
123
}
162
}
124
163
125
sub index_status_recreate_required {
164
=head2 set_index_status_reindex_required
126
    my ($self, $set) = @_;
165
127
    return defined $set ?
166
Convenience method for setting index status to C<INDEX_REINDEX_REQUIRED>.
128
        $self->index_status(INDEX_STATUS_RECREATE_REQUIRED) :
167
129
        $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
168
=cut
169
170
sub set_index_status_reindex_required {
171
    my ($self) = @_;
172
    $self->index_status(INDEX_STATUS_REINDEX_REQUIRED);
130
}
173
}
131
174
175
=head2 is_index_status_reindex_required
176
177
Convenience method for checking if index status is C<INDEX_STATUS_REINDEX_REQUIRED>.
178
179
=cut
180
181
sub is_index_status_reindex_required {
182
    my ($self) = @_;
183
    return $self->index_status == INDEX_STATUS_REINDEX_REQUIRED;
184
}
185
186
=head2 set_index_status_recreate_required
187
188
Convenience method for setting index status to C<INDEX_STATUS_RECREATE_REQUIRED>.
189
190
=cut
191
192
sub set_index_status_recreate_required {
193
    my ($self) = @_;
194
    $self->index_status(INDEX_STATUS_RECREATE_REQUIRED);
195
}
196
197
=head2 is_index_status_recreate_required
198
199
Convenience method for checking if index status is C<INDEX_STATUS_RECREATE_REQUIRED>.
200
201
=cut
202
203
sub is_index_status_recreate_required {
204
    my ($self) = @_;
205
    return $self->index_status == INDEX_STATUS_RECREATE_REQUIRED;
206
}
207
208
=head2 index_status($status)
209
210
Will either set the current index status to C<$status> and return C<$status>,
211
or return the current index status if called with no arguments.
212
213
=over 4
214
215
=item C<$status>
216
217
Optional argument. If passed will set current index status to C<$status> if C<$status> is
218
a valid status. See L</CONSTANTS>.
219
220
=back
221
222
=cut
223
132
sub index_status {
224
sub index_status {
133
    my ($self, $status) = @_;
225
    my ($self, $status) = @_;
134
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
226
    my $key = 'ElasticsearchIndexStatus_' . $self->index;
Lines 150-155 sub index_status { Link Here
150
    }
242
    }
151
}
243
}
152
244
245
=head2 update_mappings
246
247
Generate Elasticsearch mappings from mappings stored in database and
248
perform a request to update Elasticsearch index mappings. Will throw an
249
error and set index status to C<INDEX_STATUS_RECREATE_REQUIRED> if update
250
failes.
251
252
=cut
253
153
sub update_mappings {
254
sub update_mappings {
154
    my ($self) = @_;
255
    my ($self) = @_;
155
    my $conf = $self->get_elasticsearch_params();
256
    my $conf = $self->get_elasticsearch_params();
Lines 166-200 sub update_mappings { Link Here
166
                }
267
                }
167
            );
268
            );
168
        } catch {
269
        } catch {
169
            $self->index_status_recreate_required(1);
270
            $self->set_index_status_recreate_required();
170
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
271
            my $reason = $_[0]->{vars}->{body}->{error}->{reason};
171
            Koha::Exceptions::Exception->throw(
272
            Koha::Exceptions::Exception->throw(
172
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
273
                error => "Unable to update mappings for index \"$conf->{index_name}\". Reason was: \"$reason\". Index needs to be recreated and reindexed",
173
            );
274
            );
174
        };
275
        };
175
    }
276
    }
176
    $self->index_status_ok(1);
277
    $self->set_index_status_ok();
177
}
278
}
178
279
179
=head2 $indexer->update_index_background($biblionums, $records)
280
=head2 update_index_background($biblionums, $records)
180
281
181
This has exactly the same API as C<update_index_background> however it'll
282
This has exactly the same API as C<update_index> however it'll
182
return immediately. It'll start a background process that does the adding.
283
return immediately. It'll start a background process that does the adding.
183
284
184
If it fails to add to Elasticsearch then it'll add to a queue that will cause
285
If it fails to add to Elasticsearch then it'll add to a queue that will cause
185
it to be updated by a regular index cron job in the future.
286
it to be updated by a regular index cron job in the future.
186
287
288
=cut
289
187
# TODO implement in the future - I don't know the best way of doing this yet.
290
# TODO implement in the future - I don't know the best way of doing this yet.
188
# If fork: make sure process group is changed so apache doesn't wait for us.
291
# If fork: make sure process group is changed so apache doesn't wait for us.
189
292
190
=cut
191
192
sub update_index_background {
293
sub update_index_background {
193
    my $self = shift;
294
    my $self = shift;
194
    $self->update_index(@_);
295
    $self->update_index(@_);
195
}
296
}
196
297
197
=head2 $indexer->delete_index($biblionums)
298
=head2 delete_index($biblionums)
198
299
199
C<$biblionums> is an arrayref of biblionumbers to delete from the index.
300
C<$biblionums> is an arrayref of biblionumbers to delete from the index.
200
301
Lines 217-239 sub delete_index { Link Here
217
    $self->store->bag->commit;
318
    $self->store->bag->commit;
218
}
319
}
219
320
220
=head2 $indexer->delete_index_background($biblionums)
321
=head2 delete_index_background($biblionums)
221
322
222
Identical to L<delete_index>, this will return immediately and start a
323
Identical to L</delete_index($biblionums)>
223
background process to do the actual deleting.
224
324
225
=cut
325
=cut
226
326
227
# TODO implement in the future
327
# TODO: Should be made async
228
229
sub delete_index_background {
328
sub delete_index_background {
230
    my $self = shift;
329
    my $self = shift;
231
    $self->delete_index(@_);
330
    $self->delete_index(@_);
232
}
331
}
233
332
234
=head2 $indexer->drop_index();
333
=head2 drop_index
235
334
236
Drops the index from the elasticsearch server.
335
Drops the index from the Elasticsearch server.
237
336
238
=cut
337
=cut
239
338
Lines 243-252 sub drop_index { Link Here
243
        my $conf = $self->get_elasticsearch_params();
342
        my $conf = $self->get_elasticsearch_params();
244
        my $elasticsearch = $self->get_elasticsearch();
343
        my $elasticsearch = $self->get_elasticsearch();
245
        $elasticsearch->indices->delete(index => $conf->{index_name});
344
        $elasticsearch->indices->delete(index => $conf->{index_name});
246
        $self->index_status_recreate_required(1);
345
        $self->set_index_status_recreate_required();
247
    }
346
    }
248
}
347
}
249
348
349
=head2 create_index
350
351
Creates the index (including mappings) on the Elasticsearch server.
352
353
=cut
354
250
sub create_index {
355
sub create_index {
251
    my ($self) = @_;
356
    my ($self) = @_;
252
    my $conf = $self->get_elasticsearch_params();
357
    my $conf = $self->get_elasticsearch_params();
Lines 261-266 sub create_index { Link Here
261
    $self->update_mappings();
366
    $self->update_mappings();
262
}
367
}
263
368
369
=head2 index_exists
370
371
Checks if index has been created on the Elasticsearch server. Returns C<1> or the
372
empty string to indicate whether index exists or not.
373
374
=cut
375
264
sub index_exists {
376
sub index_exists {
265
    my ($self) = @_;
377
    my ($self) = @_;
266
    my $conf = $self->get_elasticsearch_params();
378
    my $conf = $self->get_elasticsearch_params();
(-)a/admin/searchengine/elasticsearch/mappings.pl (-3 / +3 lines)
Lines 135-150 my @indexes; Link Here
135
135
136
for my $index_name (@index_names) {
136
for my $index_name (@index_names) {
137
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
137
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
138
    if (!$indexer->index_status_ok) {
138
    if (!$indexer->is_index_status_ok) {
139
        my $conf = $indexer->get_elasticsearch_params();
139
        my $conf = $indexer->get_elasticsearch_params();
140
        if ($indexer->index_status_reindex_required) {
140
        if ($indexer->is_index_status_reindex_required) {
141
            push @messages, {
141
            push @messages, {
142
                type => 'error',
142
                type => 'error',
143
                code => 'reindex_required',
143
                code => 'reindex_required',
144
                index => $conf->{index_name},
144
                index => $conf->{index_name},
145
            };
145
            };
146
        }
146
        }
147
        elsif($indexer->index_status_recreate_required) {
147
        elsif($indexer->is_index_status_recreate_required) {
148
            push @messages, {
148
            push @messages, {
149
                type => 'error',
149
                type => 'error',
150
                code => 'recreate_required',
150
                code => 'recreate_required',
(-)a/misc/search_tools/rebuild_elastic_search.pl (-3 / +3 lines)
Lines 165-177 sub do_reindex { Link Here
165
        $indexer->drop_index() if $indexer->index_exists();
165
        $indexer->drop_index() if $indexer->index_exists();
166
        $indexer->create_index();
166
        $indexer->create_index();
167
    }
167
    }
168
    elsif (!$indexer->index_exists()) {
168
    elsif (!$indexer->index_exists) {
169
        # Create index if does not exist
169
        # Create index if does not exist
170
        $indexer->create_index();
170
        $indexer->create_index();
171
    } elsif ($indexer->index_status_ok) {
171
    } elsif ($indexer->is_index_status_ok) {
172
        # Update mapping unless index is some kind of problematic state
172
        # Update mapping unless index is some kind of problematic state
173
        $indexer->update_mappings();
173
        $indexer->update_mappings();
174
    } elsif ($indexer->index_status_recreate_required) {
174
    } elsif ($indexer->is_index_status_recreate_required) {
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
175
        warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
176
    }
176
    }
177
177
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +26 lines)
Lines 115-121 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
115
115
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
116
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117
117
118
    plan tests => 30;
118
    plan tests => 32;
119
119
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
120
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121
121
Lines 315-320 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
315
315
316
    ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
316
    ok(defined $docs->[0][1]->{marc_format}, 'First document marc_format field should be set');
317
317
318
    is($docs->[0][1]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly');
319
318
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
320
    is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
319
    is_deeply(
321
    is_deeply(
320
        $docs->[0][1]->{type_of_record},
322
        $docs->[0][1]->{type_of_record},
Lines 351-354 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
351
353
352
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
354
    ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
353
355
356
    # Marc serialization format fallback for records exceeding ISO2709 max record size
357
358
    my $large_marc_record = MARC::Record->new();
359
    $large_marc_record->leader('     cam  22      a 4500');
360
361
    $large_marc_record->append_fields(
362
        MARC::Field->new('100', '', '', a => 'Author 1'),
363
        MARC::Field->new('110', '', '', a => 'Corp Author'),
364
        MARC::Field->new('210', '', '', a => 'Title 1'),
365
        MARC::Field->new('245', '', '', a => 'Title:', b => 'large record'),
366
        MARC::Field->new('999', '', '', c => '1234567'),
367
    );
368
369
    my $item_field = MARC::Field->new('952', '', '', o => '123456789123456789123456789', p => '123456789', z => 'test');
370
    my $items_count = 1638;
371
    while(--$items_count) {
372
        $large_marc_record->append_fields($item_field);
373
    }
374
375
    $docs = $see->marc_records_to_documents([$large_marc_record]);
376
377
    is($docs->[0][1]->{marc_format}, 'MARCXML', 'For record exceeding max record size marc_format should be set correctly');
378
354
};
379
};
355
- 

Return to bug 19893