View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

(-)a/Koha/Exceptions/Elasticsearch.pm (+17 lines)
Line 0 Link Here
1
package Koha::Exceptions::Elasticsearch;
2
3
use Modern::Perl;
4
5
use Exception::Class (
6
7
    'Koha::Exceptions::Elasticsearch' => {
8
        description => 'Something went wrong!',
9
    },
10
    'Koha::Exceptions::Elasticsearch::MARCFieldExprParseError' => {
11
        isa => 'Koha::Exceptions::Elasticsearch',
12
        description => 'Parse error while processing MARC field expression in mapping',
13
    }
14
15
);
16
17
1;
(-)a/Koha/SearchEngine/Elasticsearch.pm (-29 / +42 lines)
Lines 35-41 use Try::Tiny; Link Here
35
use YAML::Syck;
35
use YAML::Syck;
36
36
37
use List::Util qw( sum0 reduce );
37
use List::Util qw( sum0 reduce );
38
use Search::Elasticsearch;
39
use MARC::File::XML;
38
use MARC::File::XML;
40
use MIME::Base64;
39
use MIME::Base64;
41
use Encode qw(encode);
40
use Encode qw(encode);
Lines 69-75 sub new { Link Here
69
    my $class = shift @_;
68
    my $class = shift @_;
70
    my $self = $class->SUPER::new(@_);
69
    my $self = $class->SUPER::new(@_);
71
    # Check for a valid index
70
    # Check for a valid index
72
    croak('No index name provided') unless $self->index;
71
    Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index;
73
    return $self;
72
    return $self;
74
}
73
}
75
74
Lines 312-328 sub sort_fields { Link Here
312
311
313
=head2 _process_mappings($mappings, $data, $record_document)
312
=head2 _process_mappings($mappings, $data, $record_document)
314
313
315
Process all C<$mappings> targets operating on a specific MARC field C<$data> applied to C<$record_document>
314
    $self->_process_mappings($mappings, $marc_field_data, $record_document)
316
Since we group all mappings by MARC field targets C<$mappings> will contain all targets for C<$data>
315
317
and thus we need to fetch the MARC field only once.
316
Process all C<$mappings> targets operating on a specific MARC field C<$data>.
317
Since we group all mappings by MARC field targets C<$mappings> will contain
318
all targets for C<$data> and thus we need to fetch the MARC field only once.
319
C<$mappings> will be applied to C<$record_document> and new field values added.
320
The method has no return value.
318
321
319
=over 4
322
=over 4
320
323
321
=item C<$mappings>
324
=item C<$mappings>
322
325
323
Arrayref of mappings containing arrayrefs on the format [C<$taget>, C<$options>] where
326
Arrayref of mappings containing arrayrefs in the format
324
C<$target> is the name of the target field and C<$options> is a hashref containing processing
327
[C<$taget>, C<$options>] where C<$target> is the name of the target field and
325
directives for this particular mapping.
328
C<$options> is a hashref containing processing directives for this particular
329
mapping.
326
330
327
=item C<$data>
331
=item C<$data>
328
332
Lines 330-336 The source data from a MARC record field. Link Here
330
334
331
=item C<$record_document>
335
=item C<$record_document>
332
336
333
Hashref representing the  Elasticsearch document on which mappings should be applied.
337
Hashref representing the Elasticsearch document on which mappings should be
338
applied.
334
339
335
=back
340
=back
336
341
Lines 465-471 sub marc_records_to_documents { Link Here
465
            # Suppress warnings if record length exceeded
470
            # Suppress warnings if record length exceeded
466
            unless (substr($record->leader(), 0, 5) eq '99999') {
471
            unless (substr($record->leader(), 0, 5) eq '99999') {
467
                foreach my $warning (@warnings) {
472
                foreach my $warning (@warnings) {
468
                    carp($warning);
473
                    carp $warning;
469
                }
474
                }
470
            }
475
            }
471
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
476
            $record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
Lines 482-496 sub marc_records_to_documents { Link Here
482
487
483
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
488
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
484
489
485
Get mappings, an internal data structure later used by L<_process_mappings($mappings, $data, $record_document)>
490
    my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range)
486
to process MARC target data, for a MARC mapping.
487
491
488
The returned C<$mappings> is to to be confused  with mappings provided by C<_foreach_mapping>, rather this
492
Get mappings, an internal data structure later used by
489
sub accepts properties from a mapping as provided by C<_foreach_mapping> and expands it to this internal
493
L<_process_mappings($mappings, $data, $record_document)> to process MARC target
490
data stucture. In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> is then
494
data for a MARC mapping.
491
applied to each MARC target (leader, control field data, subfield or joined subfields) and
495
492
integrated into the mapping rules data structure used in C<marc_records_to_documents> to
496
The returned C<$mappings> is not to to be confused with mappings provided by
493
transform MARC records into Elasticsearch documents.
497
C<_foreach_mapping>, rather this sub accepts properties from a mapping as
498
provided by C<_foreach_mapping> and expands it to this internal data stucture.
499
In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings>
500
is then applied to each MARC target (leader, control field data, subfield or
501
joined subfields) and integrated into the mapping rules data structure used in
502
C<marc_records_to_documents> to transform MARC records into Elasticsearch
503
documents.
494
504
495
=over 4
505
=over 4
496
506
Lines 516-529 Elasticsearch document target field type. Link Here
516
526
517
=item C<$range>
527
=item C<$range>
518
528
519
An optinal range as a string on the format "<START>-<END>" or "<START>",
529
An optional range as a string in the format "<START>-<END>" or "<START>",
520
where "<START>" and "<END>" are integers specifying a range that will be used
530
where "<START>" and "<END>" are integers specifying a range that will be used
521
for extracting a substing from MARC data as Elasticsearch field target value.
531
for extracting a substring from MARC data as Elasticsearch field target value.
522
532
523
The first character position is "1", and the range is inclusive,
533
The first character position is "1", and the range is inclusive,
524
so "1-3" means the first three characters of MARC data.
534
so "1-3" means the first three characters of MARC data.
525
535
526
If only "<START>" is provided only one character as position "<START>" will
536
If only "<START>" is provided only one character at position "<START>" will
527
be extracted.
537
be extracted.
528
538
529
=back
539
=back
Lines 593-599 rules keyed by MARC field tags holding all the mapping rules for that particular Link Here
593
603
594
We can then iterate through all MARC fields for each record and apply all relevant
604
We can then iterate through all MARC fields for each record and apply all relevant
595
rules once per fields instead of retreiving fields multiple times for each mapping rule
605
rules once per fields instead of retreiving fields multiple times for each mapping rule
596
wich is terribly slow.
606
which is terribly slow.
597
607
598
=cut
608
=cut
599
609
Lines 604-613 wich is terribly slow. Link Here
604
614
605
sub _get_marc_mapping_rules {
615
sub _get_marc_mapping_rules {
606
    my ($self) = @_;
616
    my ($self) = @_;
607
608
    my $marcflavour = lc C4::Context->preference('marcflavour');
617
    my $marcflavour = lc C4::Context->preference('marcflavour');
609
    my @rules;
610
611
    my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
618
    my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
612
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
619
    my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
613
    my $rules = {
620
    my $rules = {
Lines 625-631 sub _get_marc_mapping_rules { Link Here
625
        if ($type eq 'sum') {
632
        if ($type eq 'sum') {
626
            push @{$rules->{sum}}, $name;
633
            push @{$rules->{sum}}, $name;
627
        }
634
        }
628
        elsif($type eq 'boolean') {
635
        elsif ($type eq 'boolean') {
629
            # boolean gets special handling, if value doesn't exist for a field,
636
            # boolean gets special handling, if value doesn't exist for a field,
630
            # it is set to false
637
            # it is set to false
631
            $rules->{defaults}->{$name} = 'false';
638
            $rules->{defaults}->{$name} = 'false';
Lines 644-650 sub _get_marc_mapping_rules { Link Here
644
                foreach my $token (split //, $2) {
651
                foreach my $token (split //, $2) {
645
                    if ($token eq "(") {
652
                    if ($token eq "(") {
646
                        if ($open_group) {
653
                        if ($open_group) {
647
                            die("Unmatched opening parenthesis for $marc_field");
654
                            Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw(
655
                                "Unmatched opening parenthesis for $marc_field"
656
                            );
648
                        }
657
                        }
649
                        else {
658
                        else {
650
                            $open_group = 1;
659
                            $open_group = 1;
Lines 659-665 sub _get_marc_mapping_rules { Link Here
659
                            $open_group = 0;
668
                            $open_group = 0;
660
                        }
669
                        }
661
                        else {
670
                        else {
662
                            die("Unmatched closing parenthesis for $marc_field");
671
                            Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw(
672
                                "Unmatched closing parenthesis for $marc_field"
673
                            );
663
                        }
674
                        }
664
                    }
675
                    }
665
                    elsif ($open_group) {
676
                    elsif ($open_group) {
Lines 699-705 sub _get_marc_mapping_rules { Link Here
699
            push @{$rules->{leader}}, @mappings;
710
            push @{$rules->{leader}}, @mappings;
700
        }
711
        }
701
        else {
712
        else {
702
            die("Invalid MARC field: $marc_field");
713
            Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw(
714
                "Invalid MARC field expression: $marc_field"
715
            );
703
        }
716
        }
704
    });
717
    });
705
    return $rules;
718
    return $rules;
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +1 lines)
Lines 78-84 use constant { Link Here
78
    try {
78
    try {
79
        $self->update_index($biblionums, $records);
79
        $self->update_index($biblionums, $records);
80
    } catch {
80
    } catch {
81
        die("Something whent wrong trying to update index:" .  $_[0]);
81
        die("Something went wrong trying to update index:" .  $_[0]);
82
    }
82
    }
83
83
84
Converts C<MARC::Records> C<$records> to Elasticsearch documents and performs
84
Converts C<MARC::Records> C<$records> to Elasticsearch documents and performs
(-)a/admin/searchengine/elasticsearch/mappings.pl (-1 / +1 lines)
Lines 47-53 my $schema = $database->schema; Link Here
47
47
48
my $marc_type = lc C4::Context->preference('marcflavour');
48
my $marc_type = lc C4::Context->preference('marcflavour');
49
49
50
my @index_names = ('biblios', 'authorities');
50
my @index_names = ($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
51
51
52
my $update_mappings = sub {
52
my $update_mappings = sub {
53
    for my $index_name (@index_names) {
53
    for my $index_name (@index_names) {
(-)a/installer/data/mysql/sysprefs.sql (+2 lines)
Lines 151-156 INSERT INTO systempreferences ( `variable`, `value`, `options`, `explanation`, ` Link Here
151
('DumpTemplateVarsIntranet',  '0', NULL ,  'If enabled, dump all Template Toolkit variable to a comment in the html source for the staff intranet.',  'YesNo'),
151
('DumpTemplateVarsIntranet',  '0', NULL ,  'If enabled, dump all Template Toolkit variable to a comment in the html source for the staff intranet.',  'YesNo'),
152
('DumpTemplateVarsOpac',  '0', NULL ,  'If enabled, dump all Template Toolkit variable to a comment in the html source for the opac.',  'YesNo'),
152
('DumpTemplateVarsOpac',  '0', NULL ,  'If enabled, dump all Template Toolkit variable to a comment in the html source for the opac.',  'YesNo'),
153
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
153
('EasyAnalyticalRecords','0','','If on, display in the catalogue screens tools to easily setup analytical record relationships','YesNo'),
154
('ElasticsearchIndexStatus_authorities', '0', 'Authorities index status', NULL, NULL),
155
('ElasticsearchIndexStatus_biblios', '0', 'Biblios index status', NULL, NULL),
154
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
156
('emailLibrarianWhenHoldIsPlaced','0',NULL,'If ON, emails the librarian whenever a hold is placed','YesNo'),
155
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
157
('EnableAdvancedCatalogingEditor','0','','Enable the Rancor advanced cataloging editor','YesNo'),
156
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
158
('EnableBorrowerFiles','0',NULL,'If enabled, allows librarians to upload and attach arbitrary files to a borrower record.','YesNo'),
(-)a/t/Koha/SearchEngine/Elasticsearch.t (-2 / +1 lines)
Lines 220-226 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
220
        }
220
        }
221
    });
221
    });
222
222
223
    my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
223
    my $see = Koha::SearchEngine::Elasticsearch->new({ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX });
224
224
225
    my $marc_record_1 = MARC::Record->new();
225
    my $marc_record_1 = MARC::Record->new();
226
    $marc_record_1->leader('     cam  22      a 4500');
226
    $marc_record_1->leader('     cam  22      a 4500');
227
- 

Return to bug 19893