Attachment #78732 for bug #19365

View | Details | Raw Unified | Return to bug 19365
Collapse All | Expand All

Lines 554-560 sub LinkBibHeadingsToAuthorities { Link Here

(-)a/C4/Biblio.pm (-2 / +5 lines)
554	'', '', "a" => "" . $field->subfield('a') );	554	'', '', "a" => "" . $field->subfield('a') );
555	map {	555	map {
556	$authfield->add_subfields( $_->[0] => $_->[1] )	556	$authfield->add_subfields( $_->[0] => $_->[1] )
557	if ( $_->[0] =~ /[A-z]/ && $_->[0] ne "a" )	557	if ( $_->[0] =~ /[A-z]/ && $_->[0] ne "a"
		558	&& C4::Heading::valid_bib_heading_subfield(
		559	$authority_type->auth_tag_to_report, $_->[0] )
		560	);
558	} $field->subfields();	561	} $field->subfields();
559	$marcrecordauth->insert_fields_ordered($authfield);	562	$marcrecordauth->insert_fields_ordered($authfield);
560		563
Lines 2970-2976 sub _koha_modify_biblio { Link Here
2970		2973
2971	$sth->execute(	2974	$sth->execute(
2972	$frameworkcode, $biblio->{'author'}, $biblio->{'title'}, $biblio->{'unititle'}, $biblio->{'notes'},	2975	$frameworkcode, $biblio->{'author'}, $biblio->{'title'}, $biblio->{'unititle'}, $biblio->{'notes'},
2973	$biblio->{'serial'}, $biblio->{'seriestitle'}, $biblio->{'copyrightdate'}, $biblio->{'abstract'}, $biblio->{'biblionumber'}	2976	$biblio->{'serial'}, $biblio->{'seriestitle'}, $biblio->{'copyrightdate'} ? int($biblio->{'copyrightdate'}) : undef, $biblio->{'abstract'}, $biblio->{'biblionumber'}
2974	) if $biblio->{'biblionumber'};	2977	) if $biblio->{'biblionumber'};
2975		2978
2976	if ( $dbh->errstr \|\| !$biblio->{'biblionumber'} ) {	2979	if ( $dbh->errstr \|\| !$biblio->{'biblionumber'} ) {

Lines 17-22 package C4::Heading; Link Here

(-)a/C4/Heading.pm (-4 / +34 lines)
17	# You should have received a copy of the GNU General Public License	17	# You should have received a copy of the GNU General Public License
18	# along with Koha; if not, see <http://www.gnu.org/licenses>.	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
19		19
		20	use Modern::Perl;
		21
20	use strict;	22	use strict;
21	use warnings;	23	use warnings;
22	use MARC::Record;	24	use MARC::Record;
Lines 170-175 sub preferred_authorities { Link Here
170	return $results;	172	return $results;
171	}	173	}
172		174
		175	=head2 valid_bib_heading_subfield
		176
		177	if (C4::Heading::valid_bib_heading_subfield('100', 'e', '')) ...
		178
		179	=cut
		180
		181	sub valid_bib_heading_subfield {
		182	my $tag = shift;
		183	my $subfield = shift;
		184	my $marcflavour = @_ ? shift : C4::Context->preference('marcflavour');
		185
		186	my $marc_handler = _marc_format_handler($marcflavour);
		187
		188	return $marc_handler->valid_bib_heading_subfield( $tag, $subfield );
		189	}
		190
173	=head1 INTERNAL METHODS	191	=head1 INTERNAL METHODS
174		192
175	=head2 _search	193	=head2 _search
Lines 200-211 sub _search { Link Here
200	# push @operator, 'is';	218	# push @operator, 'is';
201	# push @value, $self->{'thesaurus'};	219	# push @value, $self->{'thesaurus'};
202	# }	220	# }
203	require C4::AuthoritiesMarc;	221
204	return C4::AuthoritiesMarc::SearchAuthorities(	222	require Koha::SearchEngine::QueryBuilder;
		223	require Koha::SearchEngine::Search;
		224
		225	+ # Use state variables to avoid recreating the objects every time.
		226	+ # With Elasticsearch this also avoids creating a massive amount of
		227	+ # ES connectors that would eventually run out of file descriptors.
		228	state $builder = Koha::SearchEngine::QueryBuilder->new(
		229	{ index => $Koha::SearchEngine::AUTHORITIES_INDEX } );
		230	state $searcher = Koha::SearchEngine::Search->new(
		231	{index => $Koha::SearchEngine::AUTHORITIES_INDEX} );
		232
		233	my $search_query = $builder->build_authorities_query_compat(
205	\@marclist, \@and_or, \@excluding, \@operator,	234	\@marclist, \@and_or, \@excluding, \@operator,
206	\@value, 0, 20, $self->{'auth_type'},	235	\@value, $self->{'auth_type'},
207	'AuthidAsc', $skipmetadata	236	'AuthidAsc'
208	);	237	);
		238	return $searcher->search_auth_compat( $search_query, 0, 20, $skipmetadata );
209	}	239	}
210		240
211	=head1 INTERNAL FUNCTIONS	241	=head1 INTERNAL FUNCTIONS





}

=head2 valid_bib_heading_subfield

=cut

sub valid_bib_heading_subfield {
    my $self          = shift;
    my $tag           = shift;
    my $subfield      = shift;

    if ( exists $bib_heading_fields->{$tag} ) {
        return 1 if ($bib_heading_fields->{$tag}->{subfields} =~ /$subfield/);
    }
    return 0;
}

=head2 parse_heading

=cut

Lines 664-670 sub get_matches { Link Here

(-)a/C4/Matcher.pm (-3 / +9 lines)
664	#NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.	664	#NOTE: double-quote the values so you don't get a "Embedded truncation not supported" error when a term has a ? in it.
665	}	665	}
666		666
667	my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});	667	# Use state variables to avoid recreating the objects every time.
		668	# With Elasticsearch this also avoids creating a massive amount of
		669	# ES connectors that would eventually run out of file descriptors.
		670	state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::BIBLIOS_INDEX});
668	( $error, $searchresults, $total_hits ) =	671	( $error, $searchresults, $total_hits ) =
669	$searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );	672	$searcher->simple_search_compat( $query, 0, $max_matches, undef, skip_normalize => 1 );
670		673
Lines 703-710 sub get_matches { Link Here
703	push @operator, 'exact';	706	push @operator, 'exact';
704	push @value, $key;	707	push @value, $key;
705	}	708	}
706	my $builder = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});	709	# Use state variables to avoid recreating the objects every time.
707	my $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});	710	# With Elasticsearch this also avoids creating a massive amount of
		711	# ES connectors that would eventually run out of file descriptors.
		712	state $builder = Koha::SearchEngine::QueryBuilder->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
		713	state $searcher = Koha::SearchEngine::Search->new({index => $Koha::SearchEngine::AUTHORITIES_INDEX});
708	my $search_query = $builder->build_authorities_query_compat(	714	my $search_query = $builder->build_authorities_query_compat(
709	\@marclist, \@and_or, \@excluding, \@operator,	715	\@marclist, \@and_or, \@excluding, \@operator,
710	\@value, undef, 'AuthidAsc'	716	\@value, undef, 'AuthidAsc'




};

sub update_index {
    my ($self, $biblionums, $records, $commit) = @_;

    # TODO should have a separate path for dealing with a large number
    # of records at once where we use the bulk update functions in ES.
    if ($biblionums) {
        $self->_sanitise_records($biblionums, $records);
    }

    $self->bulk_index($records, $commit);
    return 1;
}

sub bulk_index {
    my ($self, $records, $commit) = @_;
    my $conf = $self->get_elasticsearch_params();
    my $elasticsearch = $self->get_elasticsearch();
    my $documents = $self->marc_records_to_documents($records);

            type => 'data', # is just hard coded in Indexer.pm?
            body => \@body
        );
        $elasticsearch->flush() if ($commit);
    }
    # TODO: handle response
    return 1;

=cut

sub update_index_background {
    my ($self, $biblionums, $records) = @_;
    $self->update_index($biblionums, $records, 0);
}

=head2 $indexer->delete_index($biblionums)

        # tears in rain...
        if ( $rec ) {
            $rec->delete_fields($rec->field('999'));
            # Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs.
            $rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum));
        }
    }
}




              if $d && ( $d ne 'asc' && $d ne 'desc' );
            $d = 'asc' unless $d;

            # TODO account for fields that don't have a 'phrase' type

            $f = $self->_sort_field($f);
            push @{ $res->{sort} }, { $f => { order => $d } };
        }
    }


                }
            }
        },
        sort => [ { $sort => { order => "asc" } } ],
    };
}


    foreach my $s ( @{ $search->{searches} } ) {
        my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
        $wh = '_all' if $wh eq '';
        if ( $op eq 'is' || $op eq '='  || $op eq 'exact' ) {

            # look for something that matches a term completely
            # note, '=' is about numerical vals. May need special handling.
            # Also, we lowercase our search because the ES
            # index lowercases its values, and term searches don't get the
            # search analyzer applied to them.
            push @query_parts, { term => {"$wh.phrase" => lc $val} };
        }
        elsif ( $op eq 'exact' ) {
            # left and right truncation, otherwise an exact phrase
            push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };
        }
        elsif ( $op eq 'start' ) {
            # startswith search, uses lowercase untokenized version of heading
            push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} };
        }
        else {
            # regular wordlist stuff

    # 'should' behaves like 'or'
    # 'must' behaves like 'and'
    # Zebra results seem to match must so using that here
    my $query = { query =>
                 { bool =>
                     { must => \@query_parts  }
                 }
             };

    # We need to add '.phrase' to all the sort headings otherwise it'll sort
    # based on the tokenised form.
    my %s;
    if ( exists $search->{sort} ) {
        foreach my $k ( keys %{ $search->{sort} } ) {
            my $f = $self->_sort_field($k);
            $s{$f} = $search->{sort}{$k};
        }
        $search->{sort} = \%s;
    }


=item operator

What form of search to do. Options are: is (phrase, no truncation, whole field
must match), = (number exact match), exact (phrase, no truncation, whole field
must match). If left blank, then word list, right truncated, anywhere is used.

=item value


    'match-heading' => 'Match-heading',
    'see-from'      => 'Match-heading-see-from',
    thesaurus       => 'Subject-heading-thesaurus',
    any             => '',
    all             => ''
};

sub build_authorities_query_compat {


    my %sort;
    my $sort_field =
        ( $orderby =~ /^Heading/ ) ? 'Heading'
      : ( $orderby =~ /^Auth/ )    ? 'Local-number'
      :                              undef;
    if ($sort_field) {
        my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';


sub _sort_field {
    my ($self, $f) = @_;

    my $mappings = $self->get_elasticsearch_mappings();
    my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';
    if (!defined $self->sort_fields()->{$f} || $self->sort_fields()->{$f}) {
        $f .= '__sort';
        # We need to add '.phrase' to text fields, otherwise it'll sort
        # based on the tokenised form.
        $f .= '.phrase' if $textField;
    } else {
        # We need to add '.raw' to text fields without a sort field, 
        # otherwise it'll sort based on the tokenised form.
        $f .= '.raw' if $textField;
    }
    return $f;
}




    my ($self, $query, $page, $count, %options) = @_;

    my $params = $self->get_elasticsearch_params();
    my %paging;
    # 20 is the default number of results per page
    $query->{size} = $count || 20;
    # ES doesn't want pages, it wants a record to start from.
    if (exists $options{offset}) {
        $query->{from} = $options{offset};
    } else {
        $page = (!defined($page) || ($page <= 0)) ? 0 : $page - 1;
        $query->{from} = $page * $query->{size};
    }
    my $elasticsearch = $self->get_elasticsearch();
        Catmandu::Store::ElasticSearch->new(
            %$params,
        )
    ) unless $self->store;
    my $results = eval {
        $elasticsearch->search(
            index => $params->{index_name},
            body => $query
        );
    };
    if ($@) {
        die $self->process_error($@);

    # opac-search expects results to be put in the
    # right place in the array, according to $offset
    my $index = $offset;
    my $hits = $results->{'hits'};
    foreach my $es_record (@{$hits->{'hits'}}) {
        $records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
    }

    # consumers of this expect a name-spaced result, we provide the default
    # configuration.
    my %result;
    $result{biblioserver}{hits} = $hits->{'total'};
    $result{biblioserver}{RECORDS} = \@records;
    return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet));
}

=head2 search_auth_compat

    my ( $results, $total ) =
      $searcher->search_auth_compat( $query, $offset, $count, $skipmetadata, %options );

This has a similar calling convention to L<search>, however it returns its
results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.

=cut

sub search_auth_compat {
    my ($self, $query, $offset, $count, $skipmetadata, %options) = @_;

    if ( !defined $offset or $offset <= 0 ) {
        $offset = 1;
    }
    # Uh, authority search uses 1-based offset..
    $options{offset} = $offset - 1;
    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $res      = $self->search($query, undef, $count, %options);

    my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
    my @records;
    my $hits = $res->{'hits'};
    foreach my $es_record ($hits->{'hits'}) {
        my $record = $es_record->[0]->{'_source'};
        my %result;

        # I wonder if these should be real values defined in the mapping
        # rather than hard-coded conversions.
        #my $record    = $_[0];
        # Handle legacy nested arrays indexed with splitting enabled.
        my $authid = $record->{ 'Local-number' }[0];
        $authid = @$authid[0] if (ref $authid eq 'ARRAY');

        $result{authid} = $authid;

        if (!defined $skipmetadata || !$skipmetadata) {
            # TODO put all this info into the record at index time so we
            # don't have to go and sort it all out now.
            my $authtypecode = $record->{authtype};
            my $rs           = $schema->resultset('AuthType')
            ->search( { authtypecode => $authtypecode } );

            # FIXME there's an assumption here that we will get a result.
            # the original code also makes an assumption that some provided

            # it's not reproduced here yet.
            my $authtype           = $rs->single;
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
            my $marc               = $self->decode_record_from_result($record);
            my $mainentry          = $marc->field($auth_tag_to_report);
            my $reported_tag;
            if ($mainentry) {


            # Reimplementing BuildSummary is out of scope because it'll be hard
            $result{summary} =
            C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
                $authtypecode );
            $result{used} = $self->count_auth_use($bib_searcher, $authid);
            push @records, \%result;
        }
        push @records, \%result;
    }
    return ( \@records, $hits->{'total'} );
}

=head2 count_auth_use




=cut

sub search_auth_compat {
    my ( $self, $q, $startfrom, $resperpage, $skipmetadata ) = @_;

    my @params = (
        @{$q}{ 'marclist', 'and_or', 'excluding', 'operator', 'value' },
        $startfrom - 1,
        $resperpage, @{$q}{ 'authtypecode', 'orderby' }, $skipmetadata
    );
    C4::AuthoritiesMarc::SearchAuthorities(@params);
}

Lines 195-203 sub process_bib { Link Here

(-)a/misc/link_bibs_to_authorities.pl (-6 / +10 lines)
195	return;	195	return;
196	}	196	}
197		197
		198	my $frameworkcode = GetFrameworkCode($biblionumber);
		199
198	my ( $headings_changed, $results ) =	200	my ( $headings_changed, $results ) =
199	LinkBibHeadingsToAuthorities( $linker, $bib,	201	LinkBibHeadingsToAuthorities( $linker, $bib, $frameworkcode );
200	GetFrameworkCode($biblionumber) );
201	foreach my $key ( keys %{ $results->{'unlinked'} } ) {	202	foreach my $key ( keys %{ $results->{'unlinked'} } ) {
202	$unlinked_headings{$key} += $results->{'unlinked'}->{$key};	203	$unlinked_headings{$key} += $results->{'unlinked'}->{$key};
203	}	204	}
Lines 211-221 sub process_bib { Link Here
211	if ($headings_changed) {	212	if ($headings_changed) {
212	if ($verbose) {	213	if ($verbose) {
213	my $title = substr( $bib->title, 0, 20 );	214	my $title = substr( $bib->title, 0, 20 );
214	print	215	printf(
215	"Bib $biblionumber ($title): $headings_changed headings changed\n";	216	"Bib %12d (%-20s): %3d headings changed\n",
		217	$biblionumber,
		218	$title,
		219	$headings_changed
		220	);
216	}	221	}
217	if ( not $test_only ) {	222	if ( not $test_only ) {
218	ModBiblio( $bib, $biblionumber, GetFrameworkCode($biblionumber) );	223	ModBiblio( $bib, $biblionumber, $frameworkcode );
219	$num_bibs_modified++;	224	$num_bibs_modified++;
220	}	225	}
221	}	226	}
222	-

Return to bug 19365

Lines 102-111 sub build_query { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (-21 / +24 lines)
102	if $d && ( $d ne 'asc' && $d ne 'desc' );	102	if $d && ( $d ne 'asc' && $d ne 'desc' );
103	$d = 'asc' unless $d;	103	$d = 'asc' unless $d;
104		104
105	# TODO account for fields that don't have a 'phrase' type
106
107	$f = $self->_sort_field($f);	105	$f = $self->_sort_field($f);
108	push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };	106	push @{ $res->{sort} }, { $f => { order => $d } };
109	}	107	}
110	}	108	}
111		109
Lines 171-177 sub build_browse_query { Link Here
171	}	169	}
172	}	170	}
173	},	171	},
174	sort => [ { "$sort.phrase" => { order => "asc" } } ],	172	sort => [ { $sort => { order => "asc" } } ],
175	};	173	};
176	}	174	}
177		175
Lines 294-315 sub build_authorities_query { Link Here
294	foreach my $s ( @{ $search->{searches} } ) {	292	foreach my $s ( @{ $search->{searches} } ) {
295	my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};	293	my ( $wh, $op, $val ) = @{$s}{qw(where operator value)};
296	$wh = '_all' if $wh eq '';	294	$wh = '_all' if $wh eq '';
297	if ( $op eq 'is' \|\| $op eq '=' ) {	295	if ( $op eq 'is' \|\| $op eq '=' \|\| $op eq 'exact' ) {
298		296
299	# look for something that matches a term completely	297	# look for something that matches a term completely
300	# note, '=' is about numerical vals. May need special handling.	298	# note, '=' is about numerical vals. May need special handling.
301	# Also, we lowercase our search because the ES	299	# Also, we lowercase our search because the ES
302	# index lowercases its values, and term searches don't get the	300	# index lowercases its values, and term searches don't get the
303	# search analyzer applied to them.	301	# search analyzer applied to them.
304	push @query_parts, { term => {"$wh.phrase" => lc $val} };
305	}
306	elsif ( $op eq 'exact' ) {
307	# left and right truncation, otherwise an exact phrase
308	push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };	302	push @query_parts, { match_phrase => {"$wh.phrase" => lc $val} };
309	}	303	}
310	elsif ( $op eq 'start' ) {	304	elsif ( $op eq 'start' ) {
311	# startswith search, uses lowercase untokenized version of heading	305	# startswith search, uses lowercase untokenized version of heading
312	push @query_parts, { prefix => {"$wh.lc_raw" => lc $val} };	306	push @query_parts, { match_phrase_prefix => {"$wh.phrase" => lc $val} };
313	}	307	}
314	else {	308	else {
315	# regular wordlist stuff	309	# regular wordlist stuff
Lines 325-343 sub build_authorities_query { Link Here
325	# 'should' behaves like 'or'	319	# 'should' behaves like 'or'
326	# 'must' behaves like 'and'	320	# 'must' behaves like 'and'
327	# Zebra results seem to match must so using that here	321	# Zebra results seem to match must so using that here
328	my $query = { query=>	322	my $query = { query =>
329	{ bool =>	323	{ bool =>
330	{ must => \@query_parts }	324	{ must => \@query_parts }
331	}	325	}
332	};	326	};
333		327
334	# We need to add '.phrase' to all the sort headings otherwise it'll sort
335	# based on the tokenised form.
336	my %s;	328	my %s;
337	if ( exists $search->{sort} ) {	329	if ( exists $search->{sort} ) {
338	foreach my $k ( keys %{ $search->{sort} } ) {	330	foreach my $k ( keys %{ $search->{sort} } ) {
339	my $f = $self->_sort_field($k);	331	my $f = $self->_sort_field($k);
340	$s{"$f.phrase"} = $search->{sort}{$k};	332	$s{$f} = $search->{sort}{$k};
341	}	333	}
342	$search->{sort} = \%s;	334	$search->{sort} = \%s;
343	}	335	}
Lines 378-386 Also ignored. Link Here
378		370
379	=item operator	371	=item operator
380		372
381	What form of search to do. Options are: is (phrase, no trunction, whole field	373	What form of search to do. Options are: is (phrase, no truncation, whole field
382	must match), = (number exact match), exact (phrase, but with left and right	374	must match), = (number exact match), exact (phrase, no truncation, whole field
383	truncation). If left blank, then word list, right truncted, anywhere is used.	375	must match). If left blank, then word list, right truncated, anywhere is used.
384		376
385	=item value	377	=item value
386		378
Lines 412-418 our $koha_to_index_name = { Link Here
412	'match-heading' => 'Match-heading',	404	'match-heading' => 'Match-heading',
413	'see-from' => 'Match-heading-see-from',	405	'see-from' => 'Match-heading-see-from',
414	thesaurus => 'Subject-heading-thesaurus',	406	thesaurus => 'Subject-heading-thesaurus',
415	all => ''	407	any => '',
		408	all => ''
416	};	409	};
417		410
418	sub build_authorities_query_compat {	411	sub build_authorities_query_compat {
Lines 441-448 sub build_authorities_query_compat { Link Here
441		434
442	my %sort;	435	my %sort;
443	my $sort_field =	436	my $sort_field =
444	( $orderby =~ /^Heading/ ) ? 'Heading__sort'	437	( $orderby =~ /^Heading/ ) ? 'Heading'
445	: ( $orderby =~ /^Auth/ ) ? 'Local-Number'	438	: ( $orderby =~ /^Auth/ ) ? 'Local-number'
446	: undef;	439	: undef;
447	if ($sort_field) {	440	if ($sort_field) {
448	my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';	441	my $sort_order = ( $orderby =~ /Asc$/ ) ? 'asc' : 'desc';
Lines 774-781 the end. Maybe it'll be something else in the future, who knows? Link Here
774		767
775	sub _sort_field {	768	sub _sort_field {
776	my ($self, $f) = @_;	769	my ($self, $f) = @_;
777	if ($self->sort_fields()->{$f}) {	770
		771	my $mappings = $self->get_elasticsearch_mappings();
		772	my $textField = defined $mappings->{data}{properties}{$f}{type} && $mappings->{data}{properties}{$f}{type} eq 'text';
		773	if (!defined $self->sort_fields()->{$f} \|\| $self->sort_fields()->{$f}) {
778	$f .= '__sort';	774	$f .= '__sort';
		775	# We need to add '.phrase' to text fields, otherwise it'll sort
		776	# based on the tokenised form.
		777	$f .= '.phrase' if $textField;
		778	} else {
		779	# We need to add '.raw' to text fields without a sort field,
		780	# otherwise it'll sort based on the tokenised form.
		781	$f .= '.raw' if $textField;
779	}	782	}
780	return $f;	783	return $f;
781	}	784	}

Lines 100-111 This passes the search query on to C4::AuthoritiesMarc::SearchAuthorities Link Here

(-)a/Koha/SearchEngine/Zebra/Search.pm (-2 / +2 lines)
100	=cut	100	=cut
101		101
102	sub search_auth_compat {	102	sub search_auth_compat {
103	my ( $self, $q, $startfrom, $resperpage ) = @_;	103	my ( $self, $q, $startfrom, $resperpage, $skipmetadata ) = @_;
104		104
105	my @params = (	105	my @params = (
106	@{$q}{ 'marclist', 'and_or', 'excluding', 'operator', 'value' },	106	@{$q}{ 'marclist', 'and_or', 'excluding', 'operator', 'value' },
107	$startfrom - 1,	107	$startfrom - 1,
108	$resperpage, @{$q}{ 'authtypecode', 'orderby' }	108	$resperpage, @{$q}{ 'authtypecode', 'orderby' }, $skipmetadata
109	);	109	);
110	C4::AuthoritiesMarc::SearchAuthorities(@params);	110	C4::AuthoritiesMarc::SearchAuthorities(@params);
111	}	111	}

Lines 161-166 sub valid_bib_heading_tag { Link Here

(-)a/C4/Heading/MARC21.pm (+15 lines)
161		161
162	}	162	}
163		163
		164	=head2 valid_bib_heading_subfield
		165
		166	=cut
		167
		168	sub valid_bib_heading_subfield {
		169	my $self = shift;
		170	my $tag = shift;
		171	my $subfield = shift;
		172
		173	if ( exists $bib_heading_fields->{$tag} ) {
		174	return 1 if ($bib_heading_fields->{$tag}->{subfields} =~ /$subfield/);
		175	}
		176	return 0;
		177	}
		178
164	=head2 parse_heading	179	=head2 parse_heading
165		180
166	=cut	181	=cut

Lines 69-88 use constant { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-8 / +8 lines)
69	};	69	};
70		70
71	sub update_index {	71	sub update_index {
72	my ($self, $biblionums, $records) = @_;	72	my ($self, $biblionums, $records, $commit) = @_;
73		73
74	# TODO should have a separate path for dealing with a large number
75	# of records at once where we use the bulk update functions in ES.
76	if ($biblionums) {	74	if ($biblionums) {
77	$self->_sanitise_records($biblionums, $records);	75	$self->_sanitise_records($biblionums, $records);
78	}	76	}
79		77
80	$self->bulk_index($records);	78	$self->bulk_index($records, $commit);
81	return 1;	79	return 1;
82	}	80	}
83		81
84	sub bulk_index {	82	sub bulk_index {
85	my ($self, $records) = @_;	83	my ($self, $records, $commit) = @_;
86	my $conf = $self->get_elasticsearch_params();	84	my $conf = $self->get_elasticsearch_params();
87	my $elasticsearch = $self->get_elasticsearch();	85	my $elasticsearch = $self->get_elasticsearch();
88	my $documents = $self->marc_records_to_documents($records);	86	my $documents = $self->marc_records_to_documents($records);
Lines 103-108 sub bulk_index { Link Here
103	type => 'data', # is just hard coded in Indexer.pm?	101	type => 'data', # is just hard coded in Indexer.pm?
104	body => \@body	102	body => \@body
105	);	103	);
		104	$elasticsearch->flush() if ($commit);
106	}	105	}
107	# TODO: handle response	106	# TODO: handle response
108	return 1;	107	return 1;
Lines 190-197 it to be updated by a regular index cron job in the future. Link Here
190	=cut	189	=cut
191		190
192	sub update_index_background {	191	sub update_index_background {
193	my $self = shift;	192	my ($self, $biblionums, $records) = @_;
194	$self->update_index(@_);	193	$self->update_index($biblionums, $records, 0);
195	}	194	}
196		195
197	=head2 $indexer->delete_index($biblionums)	196	=head2 $indexer->delete_index($biblionums)
Lines 286-292 sub _sanitise_records { Link Here
286	# tears in rain...	285	# tears in rain...
287	if ( $rec ) {	286	if ( $rec ) {
288	$rec->delete_fields($rec->field('999'));	287	$rec->delete_fields($rec->field('999'));
289	$rec->append_fields(MARC::Field->new('999','','','c' => $bibnum, 'd' => $bibnum));	288	# Make sure biblionumber is a string. Elasticsearch would consider int and string different IDs.
		289	$rec->append_fields(MARC::Field->new('999','','','c' => "" . $bibnum, 'd' => "" . $bibnum));
290	}	290	}
291	}	291	}
292	}	292	}

Lines 83-105 sub search { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-35 / +42 lines)
83	my ($self, $query, $page, $count, %options) = @_;	83	my ($self, $query, $page, $count, %options) = @_;
84		84
85	my $params = $self->get_elasticsearch_params();	85	my $params = $self->get_elasticsearch_params();
86	my %paging;
87	# 20 is the default number of results per page	86	# 20 is the default number of results per page
88	$paging{limit} = $count \|\| 20;	87	$query->{size} = $count \|\| 20;
89	# ES/Catmandu doesn't want pages, it wants a record to start from.	88	# ES doesn't want pages, it wants a record to start from.
90	if (exists $options{offset}) {	89	if (exists $options{offset}) {
91	$paging{start} = $options{offset};	90	$query->{from} = $options{offset};
92	} else {	91	} else {
93	$page = (!defined($page) \|\| ($page <= 0)) ? 0 : $page - 1;	92	$page = (!defined($page) \|\| ($page <= 0)) ? 0 : $page - 1;
94	$paging{start} = $page * $paging{limit};	93	$query->{from} = $page * $query->{size};
95	}	94	}
96	$self->store(	95	my $elasticsearch = $self->get_elasticsearch();
97	Catmandu::Store::ElasticSearch->new(
98	%$params,
99	)
100	) unless $self->store;
101	my $results = eval {	96	my $results = eval {
102	$self->store->bag->search( %$query, %paging );	97	$elasticsearch->search(
		98	index => $params->{index_name},
		99	body => $query
		100	);
103	};	101	};
104	if ($@) {	102	if ($@) {
105	die $self->process_error($@);	103	die $self->process_error($@);
Lines 162-174 sub search_compat { Link Here
162	# opac-search expects results to be put in the	160	# opac-search expects results to be put in the
163	# right place in the array, according to $offset	161	# right place in the array, according to $offset
164	my $index = $offset;	162	my $index = $offset;
165	$results->each(sub {	163	my $hits = $results->{'hits'};
166	$records[$index++] = $self->decode_record_from_result(@_);	164	foreach my $es_record (@{$hits->{'hits'}}) {
167	});	165	$records[$index++] = $self->decode_record_from_result($es_record->{'_source'});
		166	}
		167
168	# consumers of this expect a name-spaced result, we provide the default	168	# consumers of this expect a name-spaced result, we provide the default
169	# configuration.	169	# configuration.
170	my %result;	170	my %result;
171	$result{biblioserver}{hits} = $results->total;	171	$result{biblioserver}{hits} = $hits->{'total'};
172	$result{biblioserver}{RECORDS} = \@records;	172	$result{biblioserver}{RECORDS} = \@records;
173	return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet));	173	return (undef, \%result, $self->_convert_facets($results->{aggregations}, $expanded_facet));
174	}	174	}
Lines 176-182 sub search_compat { Link Here
176	=head2 search_auth_compat	176	=head2 search_auth_compat
177		177
178	my ( $results, $total ) =	178	my ( $results, $total ) =
179	$searcher->search_auth_compat( $query, $page, $count, %options );	179	$searcher->search_auth_compat( $query, $offset, $count, $skipmetadata, %options );
180		180
181	This has a similar calling convention to L<search>, however it returns its	181	This has a similar calling convention to L<search>, however it returns its
182	results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.	182	results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>.
Lines 184-215 results in a form the same as L<C4::AuthoritiesMarc::SearchAuthorities>. Link Here
184	=cut	184	=cut
185		185
186	sub search_auth_compat {	186	sub search_auth_compat {
187	my $self = shift;	187	my ($self, $query, $offset, $count, $skipmetadata, %options) = @_;
188		188
189	# TODO handle paging	189	if ( !defined $offset or $offset <= 0 ) {
		190	$offset = 1;
		191	}
		192	# Uh, authority search uses 1-based offset..
		193	$options{offset} = $offset - 1;
190	my $database = Koha::Database->new();	194	my $database = Koha::Database->new();
191	my $schema = $database->schema();	195	my $schema = $database->schema();
192	my $res = $self->search(@_);	196	my $res = $self->search($query, undef, $count, %options);
		197
193	my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});	198	my $bib_searcher = Koha::SearchEngine::Elasticsearch::Search->new({index => 'biblios'});
194	my @records;	199	my @records;
195	$res->each(	200	my $hits = $res->{'hits'};
196	sub {	201	foreach my $es_record ($hits->{'hits'}) {
197	my %result;	202	my $record = $es_record->[0]->{'_source'};
		203	my %result;
198		204
199	# I wonder if these should be real values defined in the mapping	205	# I wonder if these should be real values defined in the mapping
200	# rather than hard-coded conversions.	206	# rather than hard-coded conversions.
201	my $record = $_[0];	207	#my $record = $_[0];
202	# Handle legacy nested arrays indexed with splitting enabled.	208	# Handle legacy nested arrays indexed with splitting enabled.
203	my $authid = $record->{ 'Local-number' }[0];	209	my $authid = $record->{ 'Local-number' }[0];
204	$authid = @$authid[0] if (ref $authid eq 'ARRAY');	210	$authid = @$authid[0] if (ref $authid eq 'ARRAY');
205		211
206	$result{authid} = $authid;	212	$result{authid} = $authid;
207		213
		214	if (!defined $skipmetadata \|\| !$skipmetadata) {
208	# TODO put all this info into the record at index time so we	215	# TODO put all this info into the record at index time so we
209	# don't have to go and sort it all out now.	216	# don't have to go and sort it all out now.
210	my $authtypecode = $record->{authtype};	217	my $authtypecode = $record->{authtype};
211	my $rs = $schema->resultset('AuthType')	218	my $rs = $schema->resultset('AuthType')
212	->search( { authtypecode => $authtypecode } );	219	->search( { authtypecode => $authtypecode } );
213		220
214	# FIXME there's an assumption here that we will get a result.	221	# FIXME there's an assumption here that we will get a result.
215	# the original code also makes an assumption that some provided	222	# the original code also makes an assumption that some provided
Lines 218-224 sub search_auth_compat { Link Here
218	# it's not reproduced here yet.	225	# it's not reproduced here yet.
219	my $authtype = $rs->single;	226	my $authtype = $rs->single;
220	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";	227	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
221	my $marc = $self->decode_record_from_result(@_);	228	my $marc = $self->decode_record_from_result($record);
222	my $mainentry = $marc->field($auth_tag_to_report);	229	my $mainentry = $marc->field($auth_tag_to_report);
223	my $reported_tag;	230	my $reported_tag;
224	if ($mainentry) {	231	if ($mainentry) {
Lines 232-244 sub search_auth_compat { Link Here
232		239
233	# Reimplementing BuildSummary is out of scope because it'll be hard	240	# Reimplementing BuildSummary is out of scope because it'll be hard
234	$result{summary} =	241	$result{summary} =
235	C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},	242	C4::AuthoritiesMarc::BuildSummary( $marc, $result{authid},
236	$authtypecode );	243	$authtypecode );
237	$result{used} = $self->count_auth_use($bib_searcher, $authid);	244	$result{used} = $self->count_auth_use($bib_searcher, $authid);
238	push @records, \%result;
239	}	245	}
240	);	246	push @records, \%result;
241	return ( \@records, $res->total );	247	}
		248	return ( \@records, $hits->{'total'} );
242	}	249	}
243		250
244	=head2 count_auth_use	251	=head2 count_auth_use