View | Details | Raw Unified | Return to bug 27153
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch.pm (-10 / +36 lines)
Lines 194-200 sub get_elasticsearch_mappings { Link Here
194
        my $marcflavour = lc C4::Context->preference('marcflavour');
194
        my $marcflavour = lc C4::Context->preference('marcflavour');
195
        $self->_foreach_mapping(
195
        $self->_foreach_mapping(
196
            sub {
196
            sub {
197
                my ( $name, $type, $facet, $suggestible, $sort, $search, $marc_type ) = @_;
197
                my ( $name, $type, $facet, $suggestible, $sort, $search, $filter, $marc_type ) = @_;
198
                return if $marc_type ne $marcflavour;
198
                return if $marc_type ne $marcflavour;
199
                # TODO if this gets any sort of complexity to it, it should
199
                # TODO if this gets any sort of complexity to it, it should
200
                # be broken out into its own function.
200
                # be broken out into its own function.
Lines 291-297 sub raw_elasticsearch_mappings { Link Here
291
                    marc_type => $marc_map->marc_type,
291
                    marc_type => $marc_map->marc_type,
292
                    marc_field => $marc_map->marc_field,
292
                    marc_field => $marc_map->marc_field,
293
                    sort        => $marc_to_field->sort,
293
                    sort        => $marc_to_field->sort,
294
                    suggestible => $marc_to_field->suggestible || ''
294
                    suggestible => $marc_to_field->suggestible || '',
295
                    filter => $marc_to_field->filter || ''
295
                });
296
                });
296
297
297
        }
298
        }
Lines 381-387 sub reset_elasticsearch_mappings { Link Here
381
                    facet => $mapping->{facet} || 0,
382
                    facet => $mapping->{facet} || 0,
382
                    suggestible => $mapping->{suggestible} || 0,
383
                    suggestible => $mapping->{suggestible} || 0,
383
                    sort => $mapping->{sort} // 1,
384
                    sort => $mapping->{sort} // 1,
384
                    search => $mapping->{search} // 1
385
                    search => $mapping->{search} // 1,
386
                    filter => $mapping->{filter} // ''
385
                });
387
                });
386
            }
388
            }
387
        }
389
        }
Lines 887-895 sub _array_to_marc { Link Here
887
    return $record;
889
    return $record;
888
}
890
}
889
891
890
=head2 _field_mappings($facet, $suggestible, $sort, $search, $target_name, $target_type, $range)
892
=head2 _field_mappings($facet, $suggestible, $sort, $search, $filter, $target_name, $target_type, $range)
891
893
892
    my @mappings = _field_mappings($facet, $suggestible, $sort, $search, $target_name, $target_type, $range)
894
    my @mappings = _field_mappings( $facet, $suggestible, $sort, $search, $filter, $target_name, $target_type, $range )
893
895
894
Get mappings, an internal data structure later used by
896
Get mappings, an internal data structure later used by
895
L<_process_mappings($mappings, $data, $record_document, $meta)> to process MARC target
897
L<_process_mappings($mappings, $data, $record_document, $meta)> to process MARC target
Lines 947-953 be extracted. Link Here
947
=cut
949
=cut
948
950
949
sub _field_mappings {
951
sub _field_mappings {
950
    my ($_self, $facet, $suggestible, $sort, $search, $target_name, $target_type, $range) = @_;
952
    my ( $_self, $facet, $suggestible, $sort, $search, $filter, $target_name, $target_type, $range ) = @_;
951
    my %mapping_defaults = ();
953
    my %mapping_defaults = ();
952
    my @mappings;
954
    my @mappings;
953
955
Lines 984-989 sub _field_mappings { Link Here
984
        };
986
        };
985
    }
987
    }
986
988
989
    if ( defined $filter && $filter eq 'punctuation' ) {
990
        $default_options->{value_callbacks} //= [];
991
        push @{ $default_options->{value_callbacks} }, sub {
992
            my ($value) = @_;
993
994
            # Trim punctuation marks from field
995
            $value =~
996
                s/[\x00-\x1F,\x21-\x2F,\x3A-\x40,\x5B-\x60,\x7B-\x89,\x8B,\x8D,\x8F,\x90-\x99,\x9B,\x9D,\xA0-\xBF,\xD7,\xF7]//g;
997
            return $value;
998
        };
999
    }
1000
987
    if ($search) {
1001
    if ($search) {
988
        my $mapping = [$target_name, $default_options];
1002
        my $mapping = [$target_name, $default_options];
989
        push @mappings, $mapping;
1003
        push @mappings, $mapping;
Lines 1046-1052 sub _get_marc_mapping_rules { Link Here
1046
    };
1060
    };
1047
1061
1048
    $self->_foreach_mapping(sub {
1062
    $self->_foreach_mapping(sub {
1049
        my ($name, $type, $facet, $suggestible, $sort, $search, $marc_type, $marc_field) = @_;
1063
        my ($name, $type, $facet, $suggestible, $sort, $search, $filter, $marc_type, $marc_field) = @_;
1050
        return if $marc_type ne $marcflavour;
1064
        return if $marc_type ne $marcflavour;
1051
1065
1052
        if ($type eq 'sum') {
1066
        if ($type eq 'sum') {
Lines 1110-1116 sub _get_marc_mapping_rules { Link Here
1110
            }
1124
            }
1111
1125
1112
            my $range = defined $3 ? $3 : undef;
1126
            my $range = defined $3 ? $3 : undef;
1113
            my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $name, $type, $range);
1127
            my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $filter, $name, $type, $range);
1114
            if ($field_tag < 10) {
1128
            if ($field_tag < 10) {
1115
                $rules->{control_fields}->{$field_tag} //= [];
1129
                $rules->{control_fields}->{$field_tag} //= [];
1116
                push @{$rules->{control_fields}->{$field_tag}}, @{clone(\@mappings)};
1130
                push @{$rules->{control_fields}->{$field_tag}}, @{clone(\@mappings)};
Lines 1129-1135 sub _get_marc_mapping_rules { Link Here
1129
        }
1143
        }
1130
        elsif ($marc_field =~ $leader_regexp) {
1144
        elsif ($marc_field =~ $leader_regexp) {
1131
            my $range = defined $1 ? $1 : undef;
1145
            my $range = defined $1 ? $1 : undef;
1132
            my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $name, $type, $range);
1146
            my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $search, $filter, $name, $type, $range);
1133
            push @{$rules->{leader}}, @{clone(\@mappings)};
1147
            push @{$rules->{leader}}, @{clone(\@mappings)};
1134
        }
1148
        }
1135
        else {
1149
        else {
Lines 1182-1188 sub _get_marc_mapping_rules { Link Here
1182
1196
1183
    $self->_foreach_mapping(
1197
    $self->_foreach_mapping(
1184
        sub {
1198
        sub {
1185
            my ( $name, $type, $facet, $suggestible, $sort, $marc_type,
1199
            my ( $name, $type, $facet, $suggestible, $sort, $search, $filter, $marc_type,
1186
                $marc_field )
1200
                $marc_field )
1187
              = @_;
1201
              = @_;
1188
            return unless $marc_type eq 'marc21';
1202
            return unless $marc_type eq 'marc21';
Lines 1218-1223 should be sorted on. False if a) but not b). Undef if not a). This allows, Link Here
1218
for example, author to be sorted on but not everything marked with "author"
1232
for example, author to be sorted on but not everything marked with "author"
1219
to be included in that sort.
1233
to be included in that sort.
1220
1234
1235
=item C<$search>
1236
1237
True if this value should be searchable.
1238
1239
=item C<$filter>
1240
1241
Contains a string that represents a filter defined in the indexing code. Currently supports
1242
the option 'punctuation'
1243
1221
=item C<$marc_type>
1244
=item C<$marc_type>
1222
1245
1223
A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
1246
A string that indicates the MARC type that this mapping is for, e.g. 'marc21',
Lines 1245-1250 sub _foreach_mapping { Link Here
1245
                'search_marc_to_fields.suggestible',
1268
                'search_marc_to_fields.suggestible',
1246
                'search_marc_to_fields.sort',
1269
                'search_marc_to_fields.sort',
1247
                'search_marc_to_fields.search',
1270
                'search_marc_to_fields.search',
1271
                'search_marc_to_fields.filter',
1248
                'search_marc_map.marc_type',
1272
                'search_marc_map.marc_type',
1249
                'search_marc_map.marc_field',
1273
                'search_marc_map.marc_field',
1250
            ],
1274
            ],
Lines 1253-1258 sub _foreach_mapping { Link Here
1253
                'suggestible',
1277
                'suggestible',
1254
                'sort',
1278
                'sort',
1255
                'search',
1279
                'search',
1280
                'filter',
1256
                'marc_type',
1281
                'marc_type',
1257
                'marc_field',
1282
                'marc_field',
1258
            ],
1283
            ],
Lines 1269-1274 sub _foreach_mapping { Link Here
1269
            $search_field->get_column('suggestible'),
1294
            $search_field->get_column('suggestible'),
1270
            $search_field->get_column('sort'),
1295
            $search_field->get_column('sort'),
1271
            $search_field->get_column('search'),
1296
            $search_field->get_column('search'),
1297
            $search_field->get_column('filter'),
1272
            $search_field->get_column('marc_type'),
1298
            $search_field->get_column('marc_type'),
1273
            $search_field->get_column('marc_field'),
1299
            $search_field->get_column('marc_field'),
1274
        );
1300
        );
(-)a/t/db_dependent/Koha/SearchEngine/Elasticsearch.t (-15 / +39 lines)
Lines 145-158 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
145
            marc_field => '001',
145
            marc_field => '001',
146
        },
146
        },
147
        {
147
        {
148
            name => 'isbn',
148
            name        => 'isbn',
149
            type => 'string',
149
            type        => 'string',
150
            facet => 0,
150
            facet       => 0,
151
            suggestible => 0,
151
            suggestible => 0,
152
            searchable => 1,
152
            searchable  => 1,
153
            sort => 1,
153
            filter      => 'punctuation',
154
            marc_type => 'marc21',
154
            sort        => 1,
155
            marc_field => '020a',
155
            marc_type   => 'marc21',
156
            marc_field  => '020a',
156
        },
157
        },
157
    );
158
    );
158
    my $search_engine_module = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
159
    my $search_engine_module = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
Lines 167-172 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
167
                $map->{suggestible},
168
                $map->{suggestible},
168
                $map->{sort},
169
                $map->{sort},
169
                $map->{searchable},
170
                $map->{searchable},
171
                $map->{filter},
170
                $map->{marc_type},
172
                $map->{marc_type},
171
                $map->{marc_field}
173
                $map->{marc_field}
172
            );
174
            );
Lines 184-190 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
184
186
185
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
187
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
186
188
187
    plan tests => 65;
189
    plan tests => 66;
188
190
189
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
191
    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
190
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
192
    t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', 'ISO2709');
Lines 240-245 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
240
            marc_type => 'marc21',
242
            marc_type => 'marc21',
241
            marc_field => '245(ab)ab',
243
            marc_field => '245(ab)ab',
242
        },
244
        },
245
        {
246
            name        => 'title-no-punctuation',
247
            type        => 'string',
248
            facet       => 0,
249
            suggestible => 1,
250
            searchable  => 1,
251
            sort        => undef,
252
            filter      => 'punctuation',
253
            marc_type   => 'marc21',
254
            marc_field  => '245(ab)ab',
255
        },
243
        {
256
        {
244
            name => 'unimarc_title',
257
            name => 'unimarc_title',
245
            type => 'string',
258
            type => 'string',
Lines 394-399 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
394
                $map->{suggestible},
407
                $map->{suggestible},
395
                $map->{sort},
408
                $map->{sort},
396
                $map->{searchable},
409
                $map->{searchable},
410
                $map->{filter},
397
                $map->{marc_type},
411
                $map->{marc_type},
398
                $map->{marc_field}
412
                $map->{marc_field}
399
            );
413
            );
Lines 454-464 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
454
    my $marc_record_4 = MARC::Record->new();
468
    my $marc_record_4 = MARC::Record->new();
455
    $marc_record_4->leader('     cam  22      a 4500');
469
    $marc_record_4->leader('     cam  22      a 4500');
456
    $marc_record_4->append_fields(
470
    $marc_record_4->append_fields(
457
        MARC::Field->new('008', '901111s19uu xxk|||| |00| ||eng c'),
471
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
458
        MARC::Field->new('100', '', '', a => 'Author 2'),
472
        MARC::Field->new( '100', '', '', a => 'Author 2' ),
459
        MARC::Field->new('245', '', '4', a => 'The Title :', b => 'fourth record'),
473
        MARC::Field->new( '245', '', '4', a => 'The Title\'s the thing :', b => 'fourth record' ),
460
        MARC::Field->new('260', '', '', a => 'New York :', b => 'Ace ,', c => ' 89 '),
474
        MARC::Field->new( '260', '', '', a => 'New York :', b => 'Ace ,', c => ' 89 ' ),
461
        MARC::Field->new('999', '', '', c => '1234568'),
475
        MARC::Field->new( '999', '', '', c => '1234568' ),
462
    );
476
    );
463
477
464
    my $records = [$marc_record_1, $marc_record_2, $marc_record_3, $marc_record_4];
478
    my $records = [$marc_record_1, $marc_record_2, $marc_record_3, $marc_record_4];
Lines 487-493 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
487
    is_deeply($docs->[0]->{title__sort}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
501
    is_deeply($docs->[0]->{title__sort}, ['Title: first record Title: first record'], 'First document title__sort field should be set correctly');
488
502
489
    is(scalar @{$docs->[3]->{title__sort}}, 1, 'First document title__sort field should have a single');
503
    is(scalar @{$docs->[3]->{title__sort}}, 1, 'First document title__sort field should have a single');
490
    is_deeply($docs->[3]->{title__sort}, ['Title : fourth record The Title : fourth record'], 'Fourth document title__sort field should be set correctly');
504
    is_deeply(
505
        $docs->[3]->{title__sort}, ['Title\'s the thing : fourth record The Title\'s the thing : fourth record'],
506
        'Fourth document title__sort field should be set correctly'
507
    );
508
    is_deeply(
509
        $docs->[3]->{'title-no-punctuation'},
510
        [ 'The Titles the thing ', 'fourth record', 'The Titles the thing  fourth record' ],
511
        'Fourth document title-no-punctuation field should be set correctly'
512
    );
491
513
492
    is($docs->[0]->{issues}, 6, 'Issues field should be sum of the issues for each item');
514
    is($docs->[0]->{issues}, 6, 'Issues field should be sum of the issues for each item');
493
    is($docs->[0]->{issues__sort}, 6, 'Issues sort field should also be a sum of the issues');
515
    is($docs->[0]->{issues__sort}, 6, 'Issues sort field should also be a sum of the issues');
Lines 764-769 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t Link Here
764
                $map->{suggestible},
786
                $map->{suggestible},
765
                $map->{sort},
787
                $map->{sort},
766
                $map->{searchable},
788
                $map->{searchable},
789
                $map->{filter},
767
                $map->{marc_type},
790
                $map->{marc_type},
768
                $map->{marc_field}
791
                $map->{marc_field}
769
            );
792
            );
Lines 859-864 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authori Link Here
859
                $map->{suggestible},
882
                $map->{suggestible},
860
                $map->{sort},
883
                $map->{sort},
861
                $map->{searchable},
884
                $map->{searchable},
885
                $map->{filter},
862
                $map->{marc_type},
886
                $map->{marc_type},
863
                $map->{marc_field}
887
                $map->{marc_field}
864
            );
888
            );
Lines 959-964 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu Link Here
959
                $map->{suggestible},
983
                $map->{suggestible},
960
                $map->{sort},
984
                $map->{sort},
961
                $map->{searchable},
985
                $map->{searchable},
986
                $map->{filter},
962
                $map->{marc_type},
987
                $map->{marc_type},
963
                $map->{marc_field}
988
                $map->{marc_field}
964
            );
989
            );
965
- 

Return to bug 27153