View | Details | Raw Unified | Return to bug 19365
Collapse All | Expand All

(-)a/C4/Heading.pm (-4 / +16 lines)
Lines 200-211 sub _search { Link Here
200
    #        push @operator, 'is';
200
    #        push @operator, 'is';
201
    #        push @value, $self->{'thesaurus'};
201
    #        push @value, $self->{'thesaurus'};
202
    #    }
202
    #    }
203
    require C4::AuthoritiesMarc;
203
204
    return C4::AuthoritiesMarc::SearchAuthorities(
204
    require Koha::SearchEngine::QueryBuilder;
205
    require Koha::SearchEngine::Search;
206
207
    # Use state variables to avoid recreating the objects every time.
208
    # With Elasticsearch this also avoids creating a massive amount of
209
    # ES connectors that would eventually run out of file descriptors.
210
    state $builder = Koha::SearchEngine::QueryBuilder->new(
211
        { index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
212
    state $searcher = Koha::SearchEngine::Search->new(
213
        {index => $Koha::SearchEngine::AUTHORITIES_INDEX} );
214
215
    my $search_query = $builder->build_authorities_query_compat(
205
        \@marclist, \@and_or, \@excluding, \@operator,
216
        \@marclist, \@and_or, \@excluding, \@operator,
206
        \@value,    0,        20,          $self->{'auth_type'},
217
        \@value,    $self->{'auth_type'},
207
        'AuthidAsc',         $skipmetadata
218
        'AuthidAsc'
208
    );
219
    );
220
    return $searcher->search_auth_compat( $search_query, 0, 20, $skipmetadata );
209
}
221
}
210
222
211
=head1 INTERNAL FUNCTIONS
223
=head1 INTERNAL FUNCTIONS
(-)a/C4/Matcher.pm (-3 / +9 lines)
Lines 664-670 sub get_matches { Link Here
664
                    #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
664
                    #NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
665
            }
665
            }
666
666
667
            my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
667
            # Use state variables to avoid recreating the objects every time.
668
            # With Elasticsearch this also avoids creating a massive amount of
669
            # ES connectors that would eventually run out of file descriptors.
670
            state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
668
            ( $error, $searchresults, $total_hits ) =
671
            ( $error, $searchresults, $total_hits ) =
669
              $searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
672
              $searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
670
673
Lines 703-710 sub get_matches { Link Here
703
                push @operator, 'exact';
706
                push @operator, 'exact';
704
                push @value,    $key;
707
                push @value,    $key;
705
            }
708
            }
706
            my $builder  = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
709
            # Use state variables to avoid recreating the objects every time.
707
            my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
710
            # With Elasticsearch this also avoids creating a massive amount of
711
            # ES connectors that would eventually run out of file descriptors.
712
            state $builder  = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
713
            state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
708
            my $search_query = $builder->build_authorities_query_compat(
714
            my $search_query = $builder->build_authorities_query_compat(
709
                \@marclist, \@and_or, \@excluding, \@operator,
715
                \@marclist, \@and_or, \@excluding, \@operator,
710
                \@value, undef, 'AuthidAsc'
716
                \@value, undef, 'AuthidAsc'
(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +2 lines)
Lines 398-404 sub _sanitise_records { Link Here
398
        # tears in rain...
398
        # tears in rain...
399
        if ( $rec ) {
399
        if ( $rec ) {
400
            $rec->delete_fields($rec->field('999'));
400
            $rec->delete_fields($rec->field('999'));
401
            $rec->append_fields(MARC::Field->new('999','','','c' => $bibnum, 'd' => $bibnum));
401
            # Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs.
402
            $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum));
402
        }
403
        }
403
    }
404
    }
404
}
405
}
(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-24 / +28 lines)
Lines 103-112 sub build_query { Link Here
103
              if $d && ( $d ne 'asc' && $d ne 'desc' );
103
              if $d && ( $d ne 'asc' && $d ne 'desc' );
104
            $d = 'asc' unless $d;
104
            $d = 'asc' unless $d;
105
105
106
            # TODO account for fields that don't have a 'phrase' type
107
108
            $f = $self->_sort_field($f);
106
            $f = $self->_sort_field($f);
109
            push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
107
            push @{ $res->{sort} }, { $f => { order => $d } };
110
        }
108
        }
111
    }
109
    }
112
110
Lines 172-178 sub build_browse_query { Link Here
172
                }
170
                }
173
            }
171
            }
174
        },
172
        },
175
        sort => [ { "$sort.phrase" => { order => "asc" } } ],
173
        sort => [ { $sort => { order => "asc" } } ],
176
    };
174
    };
177
}
175
}
178
176
Lines 301-322 sub build_authorities_query { Link Here
301
    foreach my $s ( @{ $search->{searches} } ) {
299
    foreach my $s ( @{ $search->{searches} } ) {
302
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
300
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
303
        $wh = '_all' if $wh eq '';
301
        $wh = '_all' if $wh eq '';
304
        if ( $op eq 'is' || $op eq '=' ) {
302
        if ( $op eq 'is' || $op eq '='  || $op eq 'exact' ) {
305
303
306
            # look for something that matches a term completely
304
            # look for something that matches a term completely
307
            # note, '=' is about numerical vals. May need special handling.
305
            # note, '=' is about numerical vals. May need special handling.
308
            # Also, we lowercase our search because the ES
306
            # Also, we lowercase our search because the ES
309
            # index lowercases its values, and term searches don't get the
307
            # index lowercases its values, and term searches don't get the
310
            # search analyzer applied to them.
308
            # search analyzer applied to them.
311
            push @query_parts, { term => {"$wh.phrase" => lc $val} };
312
        }
313
        elsif ( $op eq 'exact' ) {
314
            # left and right truncation, otherwise an exact phrase
315
            push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };
309
            push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };
316
        }
310
        }
317
        elsif ( $op eq 'start' ) {
311
        elsif ( $op eq 'start' ) {
318
            # startswith search, uses lowercase untokenized version of heading
312
            # startswith search, uses lowercase untokenized version of heading
319
            push @query_parts, { prefix => {"$wh.lc_raw" => lc $val} };
313
            push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} };
320
        }
314
        }
321
        else {
315
        else {
322
            # regular wordlist stuff
316
            # regular wordlist stuff
Lines 332-350 sub build_authorities_query { Link Here
332
    # 'should' behaves like 'or'
326
    # 'should' behaves like 'or'
333
    # 'must' behaves like 'and'
327
    # 'must' behaves like 'and'
334
    # Zebra results seem to match must so using that here
328
    # Zebra results seem to match must so using that here
335
    my $query = { query=>
329
    my $query = { query =>
336
                 { bool =>
330
                 { bool =>
337
                     { must => \@query_parts  }
331
                     { must => \@query_parts  }
338
                 }
332
                 }
339
             };
333
             };
340
334
341
    # We need to add '.phrase' to all the sort headings otherwise it'll sort
342
    # based on the tokenised form.
343
    my %s;
335
    my %s;
344
    if ( exists $search->{sort} ) {
336
    if ( exists $search->{sort} ) {
345
        foreach my $k ( keys %{ $search->{sort} } ) {
337
        foreach my $k ( keys %{ $search->{sort} } ) {
346
            my $f = $self->_sort_field($k);
338
            my $f = $self->_sort_field($k);
347
            $s{"$f.phrase"} = $search->{sort}{$k};
339
            $s{$f} = $search->{sort}{$k};
348
        }
340
        }
349
        $search->{sort} = \%s;
341
        $search->{sort} = \%s;
350
    }
342
    }
Lines 385-393 Also ignored. Link Here
385
377
386
=item operator
378
=item operator
387
379
388
What form of search to do. Options are: is (phrase, no trunction, whole field
380
What form of search to do. Options are: is (phrase, no truncation, whole field
389
must match), = (number exact match), exact (phrase, but with left and right
381
must match), = (number exact match), exact (phrase, no truncation, whole field
390
truncation). If left blank, then word list, right truncted, anywhere is used.
382
must match). If left blank, then word list, right truncated, anywhere is used.
391
383
392
=item value
384
=item value
393
385
Lines 419-425 our $koha_to_index_name = { Link Here
419
    'match-heading' => 'Match-heading',
411
    'match-heading' => 'Match-heading',
420
    'see-from'      => 'Match-heading-see-from',
412
    'see-from'      => 'Match-heading-see-from',
421
    thesaurus       => 'Subject-heading-thesaurus',
413
    thesaurus       => 'Subject-heading-thesaurus',
422
    all              => ''
414
    any             => '',
415
    all             => ''
423
};
416
};
424
417
425
sub build_authorities_query_compat {
418
sub build_authorities_query_compat {
Lines 448-455 sub build_authorities_query_compat { Link Here
448
441
449
    my %sort;
442
    my %sort;
450
    my $sort_field =
443
    my $sort_field =
451
        ( $orderby =~ /^Heading/ ) ? 'Heading__sort'
444
        ( $orderby =~ /^Heading/ ) ? 'Heading'
452
      : ( $orderby =~ /^Auth/ )    ? 'Local-Number'
445
      : ( $orderby =~ /^Auth/ )    ? 'Local-number'
453
      :                              undef;
446
      :                              undef;
454
    if ($sort_field) {
447
    if ($sort_field) {
455
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
448
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
Lines 773-788 sub _fix_limit_special_cases { Link Here
773
766
774
    my $field = $self->_sort_field($field);
767
    my $field = $self->_sort_field($field);
775
768
776
Given a field name, this works out what the actual name of the version to sort
769
Given a field name, this works out what the actual name of the field to sort
777
on should be. Often it's the same, sometimes it involves sticking "__sort" on
770
on should be. A '__sort' suffix is added for fields with a sort version, and
778
the end. Maybe it'll be something else in the future, who knows?
771
for text fields either '.phrase' (for sortable versions) or '.raw' is appended
772
to avoid sorting on a tokenized value.
779
773
780
=cut
774
=cut
781
775
782
sub _sort_field {
776
sub _sort_field {
783
    my ($self, $f) = @_;
777
    my ($self, $f) = @_;
784
    if ($self->sort_fields()->{$f}) {
778
779
    my $mappings = $self->get_elasticsearch_mappings();
780
    my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';
781
    if (!defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f}) {
785
        $f .= '__sort';
782
        $f .= '__sort';
783
        # We need to add '.phrase' to text fields, otherwise it'll sort
784
        # based on the tokenised form.
785
        $f .= '.phrase' if $textField;
786
    } else {
787
        # We need to add '.raw' to text fields without a sort field,
788
        # otherwise it'll sort based on the tokenised form.
789
        $f .= '.raw' if $textField;
786
    }
790
    }
787
    return $f;
791
    return $f;
788
}
792
}
(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-35 / +42 lines)
Lines 84-106 sub search { Link Here
84
    my ($self, $query, $page, $count, %options) = @_;
84
    my ($self, $query, $page, $count, %options) = @_;
85
85
86
    my $params = $self->get_elasticsearch_params();
86
    my $params = $self->get_elasticsearch_params();
87
    my %paging;
88
    # 20 is the default number of results per page
87
    # 20 is the default number of results per page
89
    $paging{limit} = $count || 20;
88
    $query->{size} = $count || 20;
90
    # ES/Catmandu doesn't want pages, it wants a record to start from.
89
    # ES doesn't want pages, it wants a record to start from.
91
    if (exists $options{offset}) {
90
    if (exists $options{offset}) {
92
        $paging{start} = $options{offset};
91
        $query->{from} = $options{offset};
93
    } else {
92
    } else {
94
        $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
93
        $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
95
        $paging{start} = $page * $paging{limit};
94
        $query->{from} = $page * $query->{size};
96
    }
95
    }
97
    $self->store(
96
    my $elasticsearch = $self->get_elasticsearch();
98
        Catmandu::Store::ElasticSearch->new(
99
            %$params,
100
        )
101
    ) unless $self->store;
102
    my $results = eval {
97
    my $results = eval {
103
        $self->store->bag->search( %$query, %paging );
98
        $elasticsearch->search(
99
            index => $params->{index_name},
100
            body => $query
101
        );
104
    };
102
    };
105
    if ($@) {
103
    if ($@) {
106
        die $self->process_error($@);
104
        die $self->process_error($@);
Lines 163-175 sub search_compat { Link Here
163
    # opac-search expects results to be put in the
161
    # opac-search expects results to be put in the
164
    # right place in the array, according to $offset
162
    # right place in the array, according to $offset
165
    my $index = $offset;
163
    my $index = $offset;
166
    $results->each(sub {
164
    my $hits = $results->{'hits'};
167
        $records[$index++] = $self->decode_record_from_result(@_);
165
    foreach my $es_record (@{$hits->{'hits'}}) {
168
    });
166
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
167
    }
168
169
    # consumers of this expect a name-spaced result, we provide the default
169
    # consumers of this expect a name-spaced result, we provide the default
170
    # configuration.
170
    # configuration.
171
    my %result;
171
    my %result;
172
    $result{biblioserver}{hits} = $results->total;
172
    $result{biblioserver}{hits} = $hits->{'total'};
173
    $result{biblioserver}{RECORDS} = \@records;
173
    $result{biblioserver}{RECORDS} = \@records;
174
    return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet));
174
    return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet));
175
}
175
}
Lines 177-183 sub search_compat { Link Here
177
=head2 search_auth_compat
177
=head2 search_auth_compat
178
178
179
    my ( $results, $total ) =
179
    my ( $results, $total ) =
180
      $searcher->search_auth_compat( $query, $page, $count, %options );
180
      $searcher->search_auth_compat( $query, $offset, $count, $skipmetadata, %options );
181
181
182
This has a similar calling convention to L<search>, however it returns its
182
This has a similar calling convention to L<search>, however it returns its
183
results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
183
results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
Lines 185-216 results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>. Link Here
185
=cut
185
=cut
186
186
187
sub search_auth_compat {
187
sub search_auth_compat {
188
    my $self = shift;
188
    my ($self, $query, $offset, $count, $skipmetadata, %options) = @_;
189
189
190
    # TODO handle paging
190
    if ( !defined $offset or $offset <= 0 ) {
191
        $offset = 1;
192
    }
193
    # Uh, authority search uses 1-based offset..
194
    $options{offset} = $offset - 1;
191
    my $database = Koha::Database->new();
195
    my $database = Koha::Database->new();
192
    my $schema   = $database->schema();
196
    my $schema   = $database->schema();
193
    my $res      = $self->search(@_);
197
    my $res      = $self->search($query, undef, $count, %options);
198
194
    my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
199
    my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
195
    my @records;
200
    my @records;
196
    $res->each(
201
    my $hits = $res->{'hits'};
197
        sub {
202
    foreach my $es_record (@{$hits->{'hits'}}) {
198
            my %result;
203
        my $record = $es_record->{'_source'};
204
        my %result;
199
205
200
            # I wonder if these should be real values defined in the mapping
206
        # I wonder if these should be real values defined in the mapping
201
            # rather than hard-coded conversions.
207
        # rather than hard-coded conversions.
202
            my $record    = $_[0];
208
        #my $record    = $_[0];
203
            # Handle legacy nested arrays indexed with splitting enabled.
209
        # Handle legacy nested arrays indexed with splitting enabled.
204
            my $authid = $record->{ 'Local-number' }[0];
210
        my $authid = $record->{ 'Local-number' }[0];
205
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');
211
        $authid = @$authid[0] if (ref $authid eq 'ARRAY');
206
212
207
            $result{authid} = $authid;
213
        $result{authid} = $authid;
208
214
215
        if (!defined $skipmetadata || !$skipmetadata) {
209
            # TODO put all this info into the record at index time so we
216
            # TODO put all this info into the record at index time so we
210
            # don't have to go and sort it all out now.
217
            # don't have to go and sort it all out now.
211
            my $authtypecode = $record->{authtype};
218
            my $authtypecode = $record->{authtype};
212
            my $rs           = $schema->resultset('AuthType')
219
            my $rs           = $schema->resultset('AuthType')
213
              ->search( { authtypecode => $authtypecode } );
220
            ->search( { authtypecode => $authtypecode } );
214
221
215
            # FIXME there's an assumption here that we will get a result.
222
            # FIXME there's an assumption here that we will get a result.
216
            # the original code also makes an assumption that some provided
223
            # the original code also makes an assumption that some provided
Lines 219-225 sub search_auth_compat { Link Here
219
            # it's not reproduced here yet.
226
            # it's not reproduced here yet.
220
            my $authtype           = $rs->single;
227
            my $authtype           = $rs->single;
221
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
228
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
222
            my $marc               = $self->decode_record_from_result(@_);
229
            my $marc               = $self->decode_record_from_result($record);
223
            my $mainentry          = $marc->field($auth_tag_to_report);
230
            my $mainentry          = $marc->field($auth_tag_to_report);
224
            my $reported_tag;
231
            my $reported_tag;
225
            if ($mainentry) {
232
            if ($mainentry) {
Lines 233-245 sub search_auth_compat { Link Here
233
240
234
            # Reimplementing BuildSummary is out of scope because it'll be hard
241
            # Reimplementing BuildSummary is out of scope because it'll be hard
235
            $result{summary} =
242
            $result{summary} =
236
              C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
243
            C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
237
                $authtypecode );
244
                $authtypecode );
238
            $result{used} = $self->count_auth_use($bib_searcher, $authid);
245
            $result{used} = $self->count_auth_use($bib_searcher, $authid);
239
            push @records, \%result;
240
        }
246
        }
241
    );
247
        push @records, \%result;
242
    return ( \@records, $res->total );
248
    }
249
    return ( \@records, $hits->{'total'} );
243
}
250
}
244
251
245
=head2 count_auth_use
252
=head2 count_auth_use
(-)a/Koha/SearchEngine/Zebra/Search.pm (-2 / +2 lines)
Lines 100-111 This passes the search query on to C4::AuthoritiesMarc::SearchAuthorities Link Here
100
=cut
100
=cut
101
101
102
sub search_auth_compat {
102
sub search_auth_compat {
103
    my ( $self, $q, $startfrom, $resperpage ) = @_;
103
    my ( $self, $q, $startfrom, $resperpage, $skipmetadata ) = @_;
104
104
105
    my @params = (
105
    my @params = (
106
        @{$q}{ 'marclist', 'and_or', 'excluding', 'operator', 'value' },
106
        @{$q}{ 'marclist', 'and_or', 'excluding', 'operator', 'value' },
107
        $startfrom - 1,
107
        $startfrom - 1,
108
        $resperpage, @{$q}{ 'authtypecode', 'orderby' }
108
        $resperpage, @{$q}{ 'authtypecode', 'orderby' }, $skipmetadata
109
    );
109
    );
110
    C4::AuthoritiesMarc::SearchAuthorities(@params);
110
    C4::AuthoritiesMarc::SearchAuthorities(@params);
111
}
111
}
(-)a/misc/link_bibs_to_authorities.pl (-6 / +10 lines)
Lines 195-203 sub process_bib { Link Here
195
        return;
195
        return;
196
    }
196
    }
197
197
198
    my $frameworkcode = GetFrameworkCode($biblionumber);
199
198
    my ( $headings_changed, $results ) =
200
    my ( $headings_changed, $results ) =
199
      LinkBibHeadingsToAuthorities( $linker, $bib,
201
      LinkBibHeadingsToAuthorities( $linker, $bib, $frameworkcode );
200
        GetFrameworkCode($biblionumber) );
201
    foreach my $key ( keys %{ $results->{'unlinked'} } ) {
202
    foreach my $key ( keys %{ $results->{'unlinked'} } ) {
202
        $unlinked_headings{$key} += $results->{'unlinked'}->{$key};
203
        $unlinked_headings{$key} += $results->{'unlinked'}->{$key};
203
    }
204
    }
Lines 211-221 sub process_bib { Link Here
211
    if ($headings_changed) {
212
    if ($headings_changed) {
212
        if ($verbose) {
213
        if ($verbose) {
213
            my $title = substr( $bib->title, 0, 20 );
214
            my $title = substr( $bib->title, 0, 20 );
214
            print
215
            printf(
215
"Bib $biblionumber ($title): $headings_changed headings changed\n";
216
                "Bib %12d (%-20s): %3d headings changed\n",
217
                $biblionumber,
218
                $title,
219
                $headings_changed
220
            );
216
        }
221
        }
217
        if ( not $test_only ) {
222
        if ( not $test_only ) {
218
            ModBiblio( $bib, $biblionumber, GetFrameworkCode($biblionumber) );
223
            ModBiblio( $bib, $biblionumber, $frameworkcode );
219
            $num_bibs_modified++;
224
            $num_bibs_modified++;
220
        }
225
        }
221
    }
226
    }
222
- 

Return to bug 19365