Attachment #31128 for bug #12478

View | Details | Raw Unified | Return to bug 12478
Collapse All | Expand All

Lines 3400-3405 sub ModBiblioMarc { Link Here

(-)a/C4/Biblio.pm (+8 lines)
3400	$sth = $dbh->prepare("UPDATE biblioitems SET marc=?,marcxml=? WHERE biblionumber=?");	3400	$sth = $dbh->prepare("UPDATE biblioitems SET marc=?,marcxml=? WHERE biblionumber=?");
3401	$sth->execute( $record->as_usmarc(), $record->as_xml_record($encoding), $biblionumber );	3401	$sth->execute( $record->as_usmarc(), $record->as_xml_record($encoding), $biblionumber );
3402	$sth->finish;	3402	$sth->finish;
		3403	if ( C4::Context->preference('SearchEngine') eq 'ElasticSearch' ) {
		3404	# shift to its on sub, so it can do it realtime or queue
		3405	can_load( modules => { 'Koha::ElasticSearch::Indexer' => undef } );
		3406	# need to get this from syspref probably biblio/authority for index
		3407	my $indexer = Koha::ElasticSearch::Indexer->new();
		3408	my $records = [$record];
		3409	$indexer->update_index([$biblionumber], $records);
		3410	}
3403	ModZebra( $biblionumber, "specialUpdate", "biblioserver" );	3411	ModZebra( $biblionumber, "specialUpdate", "biblioserver" );
3404	return $biblionumber;	3412	return $biblionumber;
3405	}	3413	}

Lines 2392-2400 sub _ZOOM_event_loop { Link Here

(-)a/C4/Search.pm (-2 / +8 lines)
2392	}	2392	}
2393	}	2393	}
2394		2394
2395	=head2 new_record_from_zebra	2395	=head2 new_record_from_searchengine
2396		2396
2397	Given raw data from a Zebra result set, return a MARC::Record object	2397	Given raw data from a searchengine result set, return a MARC::Record object
2398		2398
2399	This helper function is needed to take into account all the involved	2399	This helper function is needed to take into account all the involved
2400	system preferences and configuration variables to properly create the	2400	system preferences and configuration variables to properly create the
Lines 2403-2408 MARC::Record object. Link Here
2403	If we are using GRS-1, then the raw data we get from Zebra should be USMARC	2403	If we are using GRS-1, then the raw data we get from Zebra should be USMARC
2404	data. If we are using DOM, then it has to be MARCXML.	2404	data. If we are using DOM, then it has to be MARCXML.
2405		2405
		2406	If we are using elasticsearch, it'll already be a MARC::Record.
		2407
2406	=cut	2408	=cut
2407		2409
2408	sub new_record_from_zebra {	2410	sub new_record_from_zebra {
Lines 2413-2418 sub new_record_from_zebra { Link Here
2413	my $index_mode = ( $server eq 'biblioserver' )	2415	my $index_mode = ( $server eq 'biblioserver' )
2414	? C4::Context->config('zebra_bib_index_mode') // 'grs1'	2416	? C4::Context->config('zebra_bib_index_mode') // 'grs1'
2415	: C4::Context->config('zebra_auth_index_mode') // 'dom';	2417	: C4::Context->config('zebra_auth_index_mode') // 'dom';
		2418	my $search_engine = C4::Context->preference("SearchEngine");
		2419	if ($search_engine eq 'Elasticsearch') {
		2420	return $raw_data;
		2421	}
2416		2422
2417	my $marc_record = eval {	2423	my $marc_record = eval {
2418	if ( $index_mode eq 'dom' ) {	2424	if ( $index_mode eq 'dom' ) {




package Koha::Biblio;

# This contains functions to do with managing biblio records.

# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

=head1 NAME

Koha::Biblio - contains fundamental biblio-related functions

=head1 DESCRIPTION

This contains functions for normal operations on biblio records.

Note: really, C4::Biblio does the main functions, but the Koha namespace is
the new thing that should be used.

=cut

use C4::Biblio; # EmbedItemsInMarcBiblio
use Koha::Biblio::Iterator;
use Koha::Database;
use Modern::Perl;

use base qw(Class::Accessor);

__PACKAGE__->mk_accessors(qw());

=head1 FUNCTIONS

=head2 get_all_biblios_iterator

    my $it = get_all_biblios_iterator();

This will provide an iterator object that will, one by one, provide the
MARC::Record of each biblio. This will include the item data.

The iterator is a Koha::Biblio::Iterator object.

=cut

sub get_all_biblios_iterator {
    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('Biblioitem')->search( { marc => { '!=', undef } },
        { columns => [qw/ biblionumber marc /] } );
    return Koha::Biblio::Iterator->new($rs, items => 1);
}

=head2 get_marc_biblio

    my $marc = get_marc_biblio($bibnum, %options);

This fetches the MARC::Record for the given biblio number. Nothing is returned
if the biblionumber couldn't be found (or it somehow has no MARC data.)

Options are:

=over 4

=item item_data

If set to true, item data is embedded in the record. Default is to not do this.

=back

=cut

sub get_marc_biblio {
    my ($class,$bibnum, %options) = @_;

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs =
      $schema->resultset('Biblioitem')
      ->search( { marc => { '!=', undef }, biblionumber => $bibnum },
        { columns => [qw/ marc /] } );

    my $row = $rs->next();
    return unless $row;
    my $marc = MARC::Record->new_from_usmarc($row->marc);

    # TODO implement this in this module
    C4::Biblio::EmbedItemsInMarcBiblio($marc, $bibnum) if $options{item_data};

    return $marc;
}

1;




package Koha::Biblio::Iterator;

# This contains an iterator over biblio records

# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

=head1 NAME

Koha::Biblio::Iterator - iterates over biblios provided by a DBIx::Class::ResultSet

=head1 DESCRIPTION

This provides an iterator that gives the MARC::Record of each biblio that's
returned by a L<DBIx::Class::ResultSet> that provides a C<biblionumber>, and
C<marc> or C<marcxml> column from the biblioitems table.

=head1 SYNOPSIS

  use Koha::Biblio::Iterator;
  my $rs = $schema->resultset('biblioitems');
  my $iterator = Koha::Biblio::Iterator->new($rs);
  while (my $record = $iterator->next()) {
      // do something with $record
  }

=head1 METHODS

=cut

use C4::Biblio;    # :( - for EmbedItemsInMarcBiblio

use Carp;
use MARC::Record;
use MARC::File::XML;
use Modern::Perl;

=head2 new

    my $it = new($sth, option => $value, ...);

Takes a ResultSet to iterate over, and gives you an iterator on it. Optional
options may be specified.

=head3 Options

=over 4

=item items

Set to true to include item data in the resulting MARC record.

=back

=cut

sub new {
    my ( $class, $rs, %options ) = @_;

    bless {
        rs => $rs,
        %options,
    }, $class;
}

=head2 next()

In a scalar context, provides the next MARC::Record from the ResultSet, or
C<undef> if there are no more.

In a list context it will provide ($biblionumber, $record).

=cut

sub next {
    my ($self) = @_;

    my $marc;
    my $row = $self->{rs}->next();
    return if !$row;
    if ( $row->marc ) {
        $marc = MARC::Record->new_from_usmarc( $row->marc );
    }
    elsif ( $row->marcxml ) {
        $marc = MARC::Record->new_from_xml( $row->marcxml );
    }
    else {
        confess "No marc or marcxml column returned in the request.";
    }

    my $bibnum;
    if ( $self->{items} ) {
        $bibnum = $row->get_column('biblionumber');
        confess "No biblionumber column returned in the request."
          if ( !defined($bibnum) );

        # TODO this should really be in Koha::Biblio or something similar.
        C4::Biblio::EmbedItemsInMarcBiblio( $marc, $bibnum );
    }

    if (wantarray) {
        $bibnum //= $row->get_column('biblionumber');
        confess "No biblionumber column returned in the request."
          if ( !defined($bibnum) );
        return ( $bibnum, $marc );
    }
    else {
        return $marc;
    }
}

1;





__PACKAGE__->mk_accessors(qw( ));

our $schema; # the schema is a singleton

# _new_schema
# Internal helper function (not a method!). This creates a new
# database connection from the data given in the current context, and


This database handle is cached for future use: if you call
C<$database-E<gt>schema> twice, you will get the same handle both
times. If you need a second database handle, use C<&new_schema>.
possibly C<&set_schema>.

=cut

sub schema {
    my $self = shift;
    my $sth;
    if ( defined( $schema ) && $schema->storage->connected() ) {
        return $schema;
    }

    # No database handle or it died . Create one.
    $schema = &_new_schema();

    return $schema;
}

=head2 new_schema

    return &_new_schema();
}

=head2 set_schema

  $my_schema = $database->new_schema;
  $database->set_schema($my_schema);
  ...
  $database->restore_schema;

C<&set_schema> and C<&restore_schema> work in a manner analogous to
C<&set_context> and C<&restore_context>.

C<&set_schema> saves the current database handle on a stack, then sets
the current database handle to C<$my_schema>.

C<$my_schema> is assumed to be a good database handle.

=cut

sub set_schema {
    my $self       = shift;
    my $new_schema = shift;

    # Save the current database handle on the handle stack.
    # We assume that $new_schema is all good: if the caller wants to
    # screw himself by passing an invalid handle, that's fine by
    # us.
    push @{ $self->{"schema_stack"} }, $self->{"schema"};
    $self->{"schema"} = $new_schema;
}

=head2 restore_schema

  $database->restore_schema;

Restores the database handle saved by an earlier call to
C<$database-E<gt>set_schema>.

=cut

sub restore_schema {
    my $self = shift;

    if ( $#{ $self->{"schema_stack"} } < 0 ) {

        # Stack underflow
        die "SCHEMA stack underflow";
    }

    # Pop the old database handle and set it.
    $self->{"schema"} = pop @{ $self->{"schema_stack"} };

    # FIXME - If it is determined that restore_context should
    # return something, then this function should, too.
}

=head2 EXPORT

None by default.




package Koha::ElasticSearch;

# Copyright 2013 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use base qw(Class::Accessor);

use C4::Context;
use Carp;
use Elasticsearch;
use Koha::Database;
use Modern::Perl;

use Data::Dumper;    # TODO remove

__PACKAGE__->mk_ro_accessors(qw( index ));

=head1 NAME

Koha::ElasticSearch - Base module for things using elasticsearch

=head1 ACCESSORS

=over 4

=item index

The name of the index to use, generally 'biblios' or 'authorities'.

=back

=head1 FUNCTIONS

=cut

sub new {
    my $class = shift @_;
    my $self = $class->SUPER::new(@_);
    # Check for a valid index
    croak('No index name provided') unless $self->index;
    return $self;
}

=head2 get_elasticsearch_params

    my $params = $self->get_elasticsearch_params();

This provides a hashref that contains the parameters for connecting to the
ElasicSearch servers, in the form:

    {
        'servers' => ['127.0.0.1:9200', 'anotherserver:9200'],
        'index_name' => 'koha_instance',
    }

This is configured by the following in the C<config> block in koha-conf.xml:

    <elasticsearch>
        <server>127.0.0.1:9200</server>
        <server>anotherserver:9200</server>
        <index_name>koha_instance</index_name>
    </elasticsearch>

=cut

sub get_elasticsearch_params {
    my ($self) = @_;

    # Copy the hash so that we're not modifying the original
    my $es = { %{ C4::Context->config('elasticsearch') } };
    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$es );

    # Helpfully, the multiple server lines end up in an array for us anyway
    # if there are multiple ones, but not if there's only one.
    my $server = $es->{server};
    delete $es->{server};
    if ( ref($server) eq 'ARRAY' ) {

        # store it called 'servers'
        $es->{servers} = $server;
    }
    elsif ($server) {
        $es->{servers} = [$server];
    }
    else {
        die "No elasticsearch servers were specified in koha-conf.xml.\n";
    }
    die "No elasticserver index_name was specified in koha-conf.xml.\n"
      if ( !$es->{index_name} );
    # Append the name of this particular index to our namespace
    $es->{index_name} .= '_' . $self->index;
    return $es;
}

=head2 get_elasticsearch_settings

    my $settings = $self->get_elasticsearch_settings();

This provides the settings provided to elasticsearch when an index is created.
These can do things like define tokenisation methods.

A hashref containing the settings is returned.

=cut

sub get_elasticsearch_settings {
    my ($self) = @_;

    # Ultimately this should come from a file or something, and not be
    # hardcoded.
    my $settings = {
        index => {
            analysis => {
                analyzer => {
                    analyser_phrase => {
                        tokenizer => 'keyword',
                        filter    => 'lowercase',
                    },
                    analyser_standard => {
                        tokenizer => 'standard',
                        filter    => 'lowercase',
                    }
                }
            }
        }
    };
    return $settings;
}

=head2 get_elasticsearch_mappings

    my $mappings = $self->get_elasticsearch_mappings();

This provides the mappings that get passed to elasticsearch when an index is
created.

=cut

sub get_elasticsearch_mappings {
    my ($self) = @_;

    my $mappings = {
        data => {
            properties => {
                record => {
                    store          => "yes",
                    include_in_all => "false",
                    type           => "string",
                },
            }
        }
    };
    $self->_foreach_mapping(
        sub {
            my ( undef, $name, $type, $facet ) = @_;

            # TODO if this gets any sort of complexity to it, it should
            # be broken out into its own function.

            # TODO be aware of date formats, but this requires pre-parsing
            # as ES will simply reject anything with an invalid date.
            my $es_type =
              $type eq 'boolean'
              ? 'boolean'
              : 'string';
            $mappings->{data}{properties}{$name} = {
                search_analyzer => "analyser_standard",
                index_analyzer  => "analyser_standard",
                type            => $es_type,
                fields          => {
                    phrase => {
                        search_analyzer => "analyser_phrase",
                        index_analyzer  => "analyser_phrase",
                        type            => "string"
                    },
                },
            };
            $mappings->{data}{properties}{$name}{null_value} = 0
              if $type eq 'boolean';
            if ($facet) {
                $mappings->{data}{properties}{ $name . '__facet' } = {
                    type  => "string",
                    index => "not_analyzed",
                };
            }
        }
    );
    return $mappings;
}

# Provides the rules for data conversion.
sub get_fixer_rules {
    my ($self) = @_;

    my $marcflavour = lc C4::Context->preference('marcflavour');
    my @rules;
    $self->_foreach_mapping(
        sub {
            my ( undef, $name, $type, $facet, $marcs ) = @_;
            my $field = $marcs->{$marcflavour};
            return unless defined $marcs->{$marcflavour};
            my $options = '';

            # There's a bug when using 'split' with something that
            # selects a range
            # The split makes everything into nested arrays, but that's not
            # really a big deal, ES doesn't mind.
            $options = '-split => 1' unless $field =~ m|_/| || $type eq 'sum';
            push @rules, "marc_map('$field','${name}', $options)";
            if ($facet) {
                push @rules, "marc_map('$field','${name}__facet', $options)";
            }
            if ( $type eq 'boolean' ) {

                # boolean gets special handling, basically if it doesn't exist,
                # it's added and set to false. Otherwise we can't query it.
                push @rules,
                  "unless exists('$name') add_field('$name', 0) end";
            }
            if ($type eq 'sum' ) {
                push @rules, "sum('$name')";
            }
        }
    );

    return \@rules;
}

=head2 _foreach_mapping

    $self->_foreach_mapping(
        sub {
            my ( $id, $name, $type, $facet, $marcs ) = @_;
            my $marc = $marcs->{marc21};
        }
    );

This allows you to apply a function to each entry in the elasticsearch mappings
table, in order to build the mappings for whatever is needed.

In the provided function, the files are:

=over 4

=item C<$id>

An ID number, corresponding to the entry in the database.

=item C<$name>

The field name for elasticsearch (corresponds to the 'mapping' column in the
database.

=item C<$type>

The type for this value, e.g. 'string'.

=item C<$facet>

True if this value should be facetised. This only really makes sense if the
field is understood by the facet processing code anyway.

=item C<$marc>

A hashref containing the MARC field specifiers for each MARC type. It's quite
possible for this to be undefined if there is otherwise an entry in a
different MARC form.

=back

=cut

sub _foreach_mapping {
    my ( $self, $sub ) = @_;

    # TODO use a caching framework here
    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs       = $schema->resultset('ElasticsearchMapping')->search();
    for my $row ( $rs->all ) {
        $sub->(
            $row->id,
            $row->mapping,
            $row->type,
            $row->facet,
            {
                marc21  => $row->marc21,
                unimarc => $row->unimarc,
                normarc => $row->normarc
            }
        );
    }
}

1;

__END__

=head1 AUTHOR

=over 4

=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>

=item Robin Sheat C<< <robin@catalyst.net.nz> >>

=back

=cut




package Koha::ElasticSearch::Indexer;

# Copyright 2013 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Carp;
use Modern::Perl;
use base qw(Koha::ElasticSearch);
use Data::Dumper;

# For now just marc, but we can do anything here really
use Catmandu::Importer::MARC;
use Catmandu::Store::ElasticSearch;

Koha::ElasticSearch::Indexer->mk_accessors(qw( store ));

=head1 NAME

Koha::ElasticSearch::Indexer - handles adding new records to the index

=head1 SYNOPSIS

    my $indexer = Koha::ElasticSearch::Indexer->new({ index => 'biblios' });
    $indexer->delete_index();
    $indexer->update_index(\@biblionumbers, \@records);

=head1 FUNCTIONS

=cut

=head2 $indexer->update_index($biblionums, $records);

C<$biblionums> is an arrayref containing the biblionumbers for the records.

C<$records> is an arrayref containing the L<MARC::Record>s themselves.

The values in the arrays must match up, and the 999$c value in the MARC record
will be rewritten using the values in C<$biblionums> to ensure they are correct.
If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
999$c is correct on your own then.

Note that this will modify the original record if C<$biblionums> is supplied.
If that's a problem, clone them first.

=cut

sub update_index {
    my ($self, $biblionums, $records) = @_;

    if ($biblionums) {
        $self->_sanitise_records($biblionums, $records);
    }

    my $from    = $self->_convert_marc_to_json($records);
    if ( !$self->store ) {
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
                #trace_calls => 1,
            )
        );
    }
    $self->store->bag->add_many($from);
    $self->store->bag->commit;
    return 1;
}

=head2 $indexer->delete_index();

Deletes the index from the elasticsearch server. Calling C<update_index>
after this will recreate it again.

=cut

sub delete_index {
    my ($self) = @_;

    if (!$self->store) {
        # If this index doesn't exist, this will create it. Then it'll be
        # deleted. That's not the end of the world however.
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
                #trace_calls => 1,
            )
        );
    }
    $self->store->drop();
    $self->store(undef);
}

sub _sanitise_records {
    my ($self, $biblionums, $records) = @_;

    confess "Unequal number of values in \$biblionums and \$records." if (@$biblionums != @$records);

    my $c = @$biblionums;
    for (my $i=0; $i<$c; $i++) {
        my $bibnum = $biblionums->[$i];
        my $rec = $records->[$i];
        # I've seen things you people wouldn't believe. Attack ships on fire
        # off the shoulder of Orion. I watched C-beams glitter in the dark near
        # the Tannhauser gate. MARC records where 999$c doesn't match the
        # biblionumber column. All those moments will be lost in time... like
        # tears in rain...
        $rec->delete_fields($rec->field('999'));
        $rec->append_fields(MARC::Field->new('999','','','c' => $bibnum, 'd' => $bibnum));
    }
}

sub _convert_marc_to_json {
    my $self    = shift;
    my $records = shift;
    my $importer =
      Catmandu::Importer::MARC->new( records => $records, id => '999c' );
    my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
    $importer = $fixer->fix($importer);
    return $importer;
}

1;

__END__

=head1 AUTHOR

=over 4

=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>

=item Robin Sheat C<< <robin@catalyst.net.nz> >>

=back

=cut




package Koha::ElasticSearch::Search;

# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

=head1 NAME

Koha::ElasticSearch::Search - search functions for Elasticsearch

=head1 SYNOPSIS

    my $searcher = Koha::ElasticSearch::Search->new();
    my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
    my $query = $builder->build_query('perl');
    my $results = $searcher->search($query);
    print "There were " . $results->total . " results.\n";
    $results->each(sub {
        push @hits, @_[0];
    });

=head1 METHODS

=cut

use base qw(Koha::ElasticSearch);
use Koha::ItemTypes;

use Catmandu::Store::ElasticSearch;

use Data::Dumper; #TODO remove
use Carp qw(cluck);

Koha::ElasticSearch::Search->mk_accessors(qw( store ));

=head2 search

    my $results = $searcher->search($query, $page, $count);

Run a search using the query. It'll return C<$count> results, starting at page
C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)

C<%options> is a hash containing extra options:

=over 4

=item offset

If provided, this overrides the C<$page> value, and specifies the record as
an offset (i.e. the number of the record to start with), rather than a page.

=back

=cut

sub search {
    my ($self, $query, $page, $count, %options) = @_;

    my $params = $self->get_elasticsearch_params();
    my %paging;
    $paging{limit} = $count || 20;
    # ES doesn't want pages, it wants a record to start from.
    if (exists $options{offset}) {
        $paging{start} = $options{offset};
    } else {
        $page = (!defined($page) || ($page <= 0)) ? 1 : $page - 1;
        $paging{start} = $page * $paging{limit};
    }
    $self->store(
        Catmandu::Store::ElasticSearch->new(
            %$params,
            trace_calls => 0,
        )
    );
    my $results = $self->store->bag->search( %$query, %paging );
    return $results;
}

=head2 search_compat

    my ( $error, $results, $facets ) = $search->search_compat(
        $query,            $simple_query, \@sort_by,       \@servers,
        $results_per_page, $offset,       $expanded_facet, $branches,
        $query_type,       $scan
      )

A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
that is returned in the query created by build_query_compat will probably
get ignored here.

=cut

sub search_compat {
    my (
        $self,     $query,            $simple_query, $sort_by,
        $servers,  $results_per_page, $offset,       $expanded_facet,
        $branches, $query_type,       $scan
    ) = @_;

    my %options;
    $options{offset} = $offset;
    my $results = $self->search($query, undef, $results_per_page, %options);

    # Convert each result into a MARC::Record
    my (@records, $index);
    $index = $offset; # opac-search expects results to be put in the
        # right place in the array, according to $offset
    $results->each(sub {
            # The results come in an array for some reason
            my $marc_json = @_[0]->{record};
            my $marc = $self->json2marc($marc_json);
            $records[$index++] = $marc;
        });
    # consumers of this expect a name-spaced result, we provide the default
    # configuration.
    my %result;
    $result{biblioserver}{hits} = $results->total;
    $result{biblioserver}{RECORDS} = \@records;
    return (undef, \%result, $self->_convert_facets($results->{facets}));
}

=head2 json2marc

    my $marc = $self->json2marc($marc_json);

Converts the form of marc (based on its JSON, but as a Perl structure) that
Catmandu stores into a MARC::Record object.

=cut

sub json2marc {
    my ( $self, $marcjson ) = @_;

    my $marc = MARC::Record->new();
    $marc->encoding('UTF-8');

    # fields are like:
    # [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
    # conveniently, this is the form that MARC::Field->new() likes
    foreach $field (@$marcjson) {
        next if @$field < 5;    # Shouldn't be possible, but...
        if ( $field->[0] eq 'LDR' ) {
            $marc->leader( $field->[4] );
        }
        else {
            my $marc_field = MARC::Field->new(@$field);
            $marc->append_fields($marc_field);
        }
    }
    return $marc;
}

=head2 _convert_facets

    my $koha_facets = _convert_facets($es_facets);

Converts elasticsearch facets types to the form that Koha expects.
It expects the ES facet name to match the Koha type, for example C<itype>,
C<au>, C<su-to>, etc.

=cut

sub _convert_facets {
    my ( $self, $es ) = @_;

    return undef if !$es;

    # These should correspond to the ES field names, as opposed to the CCL
    # things that zebra uses.
    my %type_to_label = (
        author   => 'Authors',
        location => 'Location',
        itype    => 'ItemTypes',
        se       => 'Series',
        subject  => 'Topics',
        'su-geo' => 'Places',
    );

    # We also have some special cases, e.g. itypes that need to show the
    # value rather than the code.
    my $itypes = Koha::ItemTypes->new();
    my %special = ( itype => sub { $itypes->get_description_for_code(@_) }, );
    my @res;
    while ( ( $type, $data ) = each %$es ) {
        next if !exists( $type_to_label{$type} );
        my $facet = {
            type_id => $type . '_id',
            expand  => $type,
            expandable => 1,    # TODO figure how that's supposed to work
            "type_label_$type_to_label{$type}" => 1,
            type_link_value                    => $type,
        };
        foreach my $term ( @{ $data->{terms} } ) {
            my $t = $term->{term};
            my $c = $term->{count};
            if ( exists( $special{$type} ) ) {
                $label = $special{$type}->($t);
            }
            else {
                $label = $t;
            }
            push @{ $facet->{facets} }, {
                facet_count       => $c,
                facet_link_value  => $t,
                facet_title_value => $t . " ($c)",
                facet_label_value => $label,    # TODO either truncate this,
                     # or make the template do it like it should anyway
                type_link_value => $type,
            };
        }
        push @res, $facet if exists $facet->{facets};
    }
    return \@res;
}


1;




package Koha::ItemType;

# This represents a single itemtype

# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

=head1 NAME

Koha::ItemType - represents a single itemtype

=head1 DESCRIPTION

This contains the data relating to a single itemtype

=head1 SYNOPSIS

    use Koha::ItemTypes;
    my $types = Koha::ItemTypes->new();
    my $type  = $types->get_itemtype('CODE');
    print $type->code, $type->description, $type->rentalcharge,
      $type->imageurl, $type->summary, $type->checkinmsg,
      $type->checkinmsgtype;

Creating an instance of C<Koha::ItemType> without using L<Koha::ItemTypes>
can be done simply by passing a hashref containing the values to C<new()>.
Note when doing this that a value for C<itemtype> will become a value for
C<code>.

=head1 FUNCTIONS

In addition to the read-only accessors mentioned above, the following functions
exist.

=cut

use Modern::Perl;

use base qw(Class::Accessor);

# TODO can we make these auto-generate from the input hash so it doesn't
# have to be updated when the database is?
__PACKAGE__->mk_ro_accessors(
    qw(code description rentalcharge imageurl
      summary checkinmsg checkinmsgtype)
);

sub new {
    my $class = shift @_;

    my %data = ( %{ $_[0] }, code => $_[0]->{itemtype} );
    my $self = $class->SUPER::new( \%data );
    return $self;
}

1;




package Koha::ItemTypes;

# This contains the item types that the system knows about.

# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

=head1 NAME

Koha::ItemTypes - handles the item types that Koha knows about

=head1 DESCRIPTION

This contains functions to access the item types.

Note that any changes that happen to the database while this object is live
may not be reflected, so best don't hold onto it for a long time

=cut

use Koha::Database;
use Koha::ItemType;
use Modern::Perl;

use Data::Dumper; # TODO remove
use base qw(Class::Accessor);

__PACKAGE__->mk_accessors(qw());

=head1 FUNCTIONS

=head2 new

    my $itypes = Koha::ItemTypes->new();

Creates a new instance of the object.

=cut

# Handled by Class::Accessor

=head2 get_itemtype

    my @itype = $itypes->get_itemtype('CODE1', 'CODE2');

This returns a L<Koha::ItemType> object for each of the provided codes. For
any that don't exist, an C<undef> is returned.

=cut

sub get_itemtype {
    my ($self, @codes) = @_;

    my $schema = Koha::Database->new()->schema();
    my @res;

    foreach my $c (@codes) {
        if (exists $self->{cached}{$c}) {
            push @res, $self->{cached}{$c};
            next;
        }
        my $rs = $schema->resultset('Itemtype')->search( { itemtype => $c } );
        my $r = $rs->next;
        if (!$r) {
            push @res, undef;
            next;
        }
        my %data = $r->get_inflated_columns;
        my $it = Koha::ItemType->new(\%data);
        push @res, $it;
        $self->{cached}{$c} = $it;
    }
    if (wantarray) {
        return @res;
    } else {
        return @res ? $res[0] : undef;
    }
}

=head2 get_description_for_code

    my $desc = $itypes->get_description_for_code($code);

This returns the description for an itemtype code. As a special case, if
there is no itemtype for this code, it'll return what it was given.

It is mostly as a convenience function rather than using L<get_itemtype>.

=cut

sub get_description_for_code {
    my ($self, $code) = @_;

    my $itype = $self->get_itemtype($code);
    return $code if !$itype;
    return $itype->description;
}

1;




use utf8;
package Koha::Schema::Result::ElasticsearchMapping;

# Created by DBIx::Class::Schema::Loader
# DO NOT MODIFY THE FIRST PART OF THIS FILE

=head1 NAME

Koha::Schema::Result::ElasticsearchMapping

=cut

use strict;
use warnings;

use base 'DBIx::Class::Core';

=head1 TABLE: C<elasticsearch_mapping>

=cut

__PACKAGE__->table("elasticsearch_mapping");

=head1 ACCESSORS

=head2 id

  data_type: 'integer'
  is_auto_increment: 1
  is_nullable: 0

=head2 mapping

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 type

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 facet

  data_type: 'tinyint'
  default_value: 0
  is_nullable: 1

=head2 marc21

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 unimarc

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=head2 normarc

  data_type: 'varchar'
  is_nullable: 1
  size: 255

=cut

__PACKAGE__->add_columns(
  "id",
  { data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
  "mapping",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "type",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "facet",
  { data_type => "tinyint", default_value => 0, is_nullable => 1 },
  "marc21",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "unimarc",
  { data_type => "varchar", is_nullable => 1, size => 255 },
  "normarc",
  { data_type => "varchar", is_nullable => 1, size => 255 },
);

=head1 PRIMARY KEY

=over 4

=item * L</id>

=back

=cut

__PACKAGE__->set_primary_key("id");


# Created by DBIx::Class::Schema::Loader v0.07040 @ 2014-06-06 16:20:16
# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uGRmWU0rshP6awyLMQYJeQ


# You can replace this text with custom code or comments, and it will be preserved on regeneration
1;




package Koha::SearchEngine::Elasticsearch::QueryBuilder;

# This file is part of Koha.
#
# Copyright 2014 Catalyst IT Ltd.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

=head1 NAME

Koha::SearchEngine::Elasticsearch::QueryBuilder - constructs elasticsearch
query objects from user-supplied queries

=head1 DESCRIPTION

This provides the functions that take a user-supplied search query, and
provides something that can be given to elasticsearch to get answers.

=head1 SYNOPSIS

    use Koha::SearchEngine::Elasticsearch;
    $builder = Koha::SearchEngine::Elasticsearch->new();
    my $simple_query = $builder->build_query("hello");
    # This is currently undocumented because the original code is undocumented
    my $adv_query = $builder->build_advanced_query($indexes, $operands, $operators);

=head1 METHODS

=cut

use base qw(Class::Accessor);
use List::MoreUtils qw/ each_array /;
use Modern::Perl;
use URI::Escape;

use Data::Dumper;    # TODO remove

=head2 build_query

    my $simple_query = $builder->build_query("hello", %options)

This will build a query that can be issued to elasticsearch from the provided
string input. This expects a lucene style search form (see
L<http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax>
for details.)

It'll make an attempt to respect the various query options.

Additional options can be provided with the C<%options> hash.

=over 4

=item sort

This should be an arrayref of hashrefs, each containing a C<field> and an
C<direction> (optional, defaults to C<asc>.) The results will be sorted
according to these values. Valid values for C<direction> are 'asc' and 'desc'.

=back

=cut

sub build_query {
    my ( $self, $query, %options ) = @_;

    my $stemming         = C4::Context->preference("QueryStemming")        || 0;
    my $auto_truncation  = C4::Context->preference("QueryAutoTruncate")    || 0;
    my $weight_fields    = C4::Context->preference("QueryWeightFields")    || 0;
    my $fuzzy_enabled    = C4::Context->preference("QueryFuzzy")           || 0;

    $query = '*' unless defined $query;

    my $res;
    $res->{query} = {
        query_string => {
            query            => $query,
            fuzziness        => $fuzzy_enabled ? 'auto' : '0',
            default_operator => "AND",
            default_field    => "_all",
        }
    };

    if ( $options{sort} ) {
        foreach my $sort ( @{ $options{sort} } ) {
            my ( $f, $d ) = @$sort{qw/ field direction /};
            die "Invalid sort direction, $d"
              if $d && ( $d ne 'asc' && $d ne 'desc' );
            $d = 'asc' unless $d;

            # TODO account for fields that don't have a 'phrase' type
            push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
        }
    }

    # See _convert_facets in Search.pm for how these get turned into
    # things that Koha can use.
    $res->{facets} = {
        author  => { terms => { field => "author__facet" } },
        subject => { terms => { field => "subject__facet" } },
        itype   => { terms => { field => "itype__facet" } },
    };
    return $res;
}

=head2 build_browse_query

    my $browse_query = $builder->build_browse_query($field, $query);

This performs a "starts with" style query on a particular field. The field
to be searched must have been indexed with an appropriate mapping as a
"phrase" subfield.

=cut

sub build_browse_query {
    my ( $self, $field, $query ) = @_;

    my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") || 0;

    return { query => '*' } if !defined $query;

    # TODO this should come from Koha::Elasticsearch
    my %field_whitelist = (
        title  => 1,
        author => 1,
    );
    $field = 'title' if !exists $field_whitelist{$field};

    my $res = {
        query => {
            match_phrase_prefix => {
                "$field.phrase" => {
                    query     => $query,
                    operator  => 'or',
                    fuzziness => $fuzzy_enabled ? 'auto' : '0',
                }
            }
        },
        sort => [ { "$field.phrase" => { order => "asc" } } ],
    };
}

=head2 build_query_compat

    my (
        $error,             $query, $simple_query, $query_cgi,
        $query_desc,        $limit, $limit_cgi,    $limit_desc,
        $stopwords_removed, $query_type
      )
      = $builder->build_query_compat( \@operators, \@operands, \@indexes,
        \@limits, \@sort_by, $scan, $lang );

This handles a search using the same api as L<C4::Search::buildQuery> does.

A very simple query will go in with C<$operands> set to ['query'], and
C<$sort_by> set to ['pubdate_dsc']. This simple case will return with
C<$query> set to something that can perform the search, C<$simple_query>
set to just the search term, C<$query_cgi> set to something that can
reproduce this search, and C<$query_desc> set to something else.

=cut

sub build_query_compat {
    my ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan,
        $lang )
      = @_;

#die Dumper ( $self, $operators, $operands, $indexes, $limits, $sort_by, $scan, $lang );
    my @sort_params  = $self->_convert_sort_fields(@$sort_by);
    my @index_params = $self->_convert_index_fields(@$indexes);
    my $limits       = $self->_fix_limit_special_cases($orig_limits);

    # Merge the indexes in with the search terms and the operands so that
    # each search thing is a handy unit.
    unshift @$operators, undef;    # The first one can't have an op
    my @search_params;
    my $ea = each_array( @$operands, @$operators, @index_params );
    while ( my ( $oand, $otor, $index ) = $ea->() ) {
        next if ( !defined($oand) || $oand eq '' );
        push @search_params, {
            operand => $self->_clean_search_term($oand),      # the search terms
            operator => defined($otor) ? uc $otor : undef,    # AND and so on
            $index ? %$index : (),
        };
    }

    # We build a string query from limits and the queries. An alternative
    # would be to pass them separately into build_query and let it build
    # them into a structured ES query itself. Maybe later, though that'd be
    # more robust.
    my $query_str = join( ' AND ',
        join( ' ', $self->_create_query_string(@search_params) ),
        $self->_join_queries( $self->_convert_index_strings(@$limits) ) );

    # If there's no query on the left, let's remove the junk left behind
    $query_str =~ s/^ AND //;
    my %options;
    $options{sort} = \@sort_params;
    my $query = $self->build_query( $query_str, %options );

    #die Dumper($query);
    # We roughly emulate the CGI parameters of the zebra query builder
    my $query_cgi = 'idx=kw&q=' . uri_escape( $operands->[0] ) if @$operands;
    my $simple_query = $operands->[0] if @$operands == 1;
    my $query_desc   = $simple_query;
    my $limit        = 'and ' . join( ' and ', @$limits );
    my $limit_cgi =
      '&limit=' . join( '&limit=', map { uri_escape($_) } @$orig_limits );
    my $limit_desc = "@$limits";

    return (
        undef,  $query,     $simple_query, $query_cgi, $query_desc,
        $limit, $limit_cgi, $limit_desc,   undef,      undef
    );
}

=head2 _convert_sort_fields

    my @sort_params = _convert_sort_fields(@sort_by)

Converts the zebra-style sort index information into elasticsearch-style.

C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
something that can be sent to L<build_query>.

=cut

sub _convert_sort_fields {
    my ( $self, @sort_by ) = @_;

    # Turn the sorting into something we care about.
    my %sort_field_convert = (
        acqdate     => 'acqdate',
        author      => 'author',
        call_number => 'callnum',
        popularity  => 'issues',
        relevance   => undef,       # default
        title       => 'title',
        pubdate     => 'pubdate',
    );
    my %sort_order_convert =
      ( qw( dsc desc ), qw( asc asc ), qw( az asc ), qw( za desc ) );

    # Convert the fields and orders, drop anything we don't know about.
    grep { $_->{field} } map {
        my ( $f, $d ) = split /_/;
        {
            field     => $sort_field_convert{$f},
            direction => $sort_order_convert{$d}
        }
    } @sort_by;
}

=head2 _convert_index_fields

    my @index_params = $self->_convert_index_fields(@indexes);

Converts zebra-style search index notation into elasticsearch-style.

C<@indexes> is an array of index names, as presented to L<build_query_compat>,
and it returns something that can be sent to L<build_query>.

B<TODO>: this will pull from the elasticsearch mappings table to figure out
types.

=cut

our %index_field_convert = (
    'kw'       => '_all',
    'ti'       => 'title',
    'au'       => 'author',
    'su'       => 'subject',
    'nb'       => 'isbn',
    'se'       => 'title-series',
    'callnum'  => 'callnum',
    'mc-itype' => 'itype',
    'ln'       => 'ln',
    'branch'   => 'homebranch',
    'fic'      => 'lf',
    'mus'      => 'rtype',
    'aud'      => 'ta',
);

sub _convert_index_fields {
    my ( $self, @indexes ) = @_;

    my %index_type_convert =
      ( __default => undef, phr => 'phrase', rtrn => 'right-truncate' );

    # Convert according to our table, drop anything that doesn't convert
    grep { $_->{field} } map {
        my ( $f, $t ) = split /,/;
        {
            field => $index_field_convert{$f},
            type  => $index_type_convert{ $t // '__default' }
        }
    } @indexes;
}

=head2 _convert_index_strings

    my @searches = $self->_convert_index_strings(@searches);

Similar to L<_convert_index_fields>, this takes strings of the form
B<field:search term> and rewrites the field from zebra-style to
elasticsearch-style. Anything it doesn't understand is returned verbatim.

=cut

sub _convert_index_strings {
    my ( $self, @searches ) = @_;

    my @res;
    foreach my $s (@searches) {
        next if $s eq '';
        my ( $field, $term ) = $s =~ /^\s*([\w,-]*?):(.*)/;
        unless ( defined($field) && defined($term) ) {
            push @res, $s;
            next;
        }
        my ($conv) = $self->_convert_index_fields($field);
        unless ( defined($conv) ) {
            push @res, $s;
            next;
        }
        push @res, $conv->{field} . ":"
          . $self->_modify_string_by_type( %$conv, operand => $term );
    }
    return @res;
}

=head2 _modify_string_by_type

    my $str = $self->_modify_string_by_type(%index_field);

If you have a search term (operand) and a type (phrase, right-truncated), this
will convert the string to have the function in lucene search terms, e.g.
wrapping quotes around it.

=cut

sub _modify_string_by_type {
    my ( $self, %idx ) = @_;

    my $type = $idx{type} || '';
    my $str = $idx{operand};
    return $str unless $str;    # Empty or undef, we can't use it.

    $str .= '*' if $type eq 'right-truncate';
    $str = '"' . $str . '"' if $type eq 'phrase';
    return $str;
}

=head2 _convert_index_strings_freeform

    my $search = $self->_convert_index_strings_freeform($search);

This is similar to L<_convert_index_strings>, however it'll search out the
things to change within the string. So it can handle strings such as
C<(su:foo) AND (su:bar)>, converting the C<su> appropriately.

=cut

sub _convert_index_strings_freeform {
    my ( $self, $search ) = @_;

    while ( my ( $zeb, $es ) = each %index_field_convert ) {
        $search =~ s/\b$zeb:/$es:/g;
    }
    return $search;
}

=head2 _join_queries

    my $query_str = $self->_join_queries(@query_parts);

This takes a list of query parts, that might be search terms on their own, or
booleaned together, or specifying fields, or whatever, wraps them in
parentheses, and ANDs them all together. Suitable for feeding to the ES
query string query.

=cut

sub _join_queries {
    my ( $self, @parts ) = @_;

    @parts = grep { defined($_) && $_ ne '' } @parts;
    return () unless @parts;
    return $parts[0] if @parts < 2;
    join ' AND ', map { "($_)" } @parts;
}

=head2 _make_phrases

    my @phrased_queries = $self->_make_phrases(@query_parts);

This takes the supplied queries and forces them to be phrases by wrapping
quotes around them. It understands field prefixes, e.g. 'subject:' and puts
the quotes outside of them if they're there.

=cut

sub _make_phrases {
    my ( $self, @parts ) = @_;
    map { s/^\s*(\w*?:)(.*)$/$1"$2"/r } @parts;
}

=head2 _create_query_string

    my @query_strings = $self->_create_query_string(@queries);

Given a list of hashrefs, it will turn them into a lucene-style query string.
The hash should contain field, type (both for the indexes), operator, and
operand.

=cut

sub _create_query_string {
    my ( $self, @queries ) = @_;

    map {
        my $otor  = $_->{operator} ? $_->{operator} . ' ' : '';
        my $field = $_->{field}    ? $_->{field} . ':'    : '';

        my $oand = $self->_modify_string_by_type(%$_);
        "$otor($field$oand)";
    } @queries;
}

=head2 _clean_search_term

    my $term = $self->_clean_search_term($term);

This cleans a search term by removing any funny characters that may upset
ES and give us an error. It also calls L<_convert_index_strings_freeform>
to ensure those parts are correct.

=cut

sub _clean_search_term {
    my ( $self, $term ) = @_;

    $term = $self->_convert_index_strings_freeform($term);
    $term =~ s/[{}]/"/g;
    return $term;
}

=head2 _fix_limit_special_cases

    my $limits = $self->_fix_limit_special_cases($limits);

This converts any special cases that the limit specifications have into things
that are more readily processable by the rest of the code.

The argument should be an arrayref, and it'll return an arrayref.

=cut

sub _fix_limit_special_cases {
    my ( $self, $limits ) = @_;

    my @new_lim;
    foreach my $l (@$limits) {

        # This is set up by opac-search.pl
        if ( $l =~ /^yr,st-numeric,ge=/ ) {
            my ( $start, $end ) =
              ( $l =~ /^yr,st-numeric,ge=(.*) and yr,st-numeric,le=(.*)$/ );
            next unless defined($start) && defined($end);
            push @new_lim, "copydate:[$start TO $end]";
        }
        elsif ( $l =~ /^yr,st-numeric=/ ) {
            my ($date) = ( $l =~ /^yr,st-numeric=(.*)$/ );
            next unless defined($date);
            push @new_lim, "copydate:$date";
        }
        elsif ( $l =~ /^available$/ ) {
            push @new_lim, 'onloan:false';
        }
        else {
            push @new_lim, $l;
        }
    }
    return \@new_lim;
}

1;

Lines 20-24 package Koha::SearchEngine::QueryBuilderRole; Link Here

(-)a/Koha/SearchEngine/QueryBuilderRole.pm (+3 lines)
20	use Moose::Role;	20	use Moose::Role;
21		21
22	requires 'build_query';	22	requires 'build_query';
		23	# The compat version should accept and return parameters in the same form as
		24	# C4::Search->buildQuery does.
		25	requires 'build_query_compat';
23		26
24	1;	27	1;

Lines 19-25 package Koha::SearchEngine::Zebra; Link Here

(-)a/Koha/SearchEngine/Zebra.pm (-1 / +2 lines)
19		19
20	use Moose;	20	use Moose;
21		21
22	extends 'Data::SearchEngine::Zebra';	22	# Removed because it doesn't exist.
		23	#extends 'Data::SearchEngine::Zebra';
23		24
24	# the configuration file is retrieved from KOHA_CONF by default, provide it from there²	25	# the configuration file is retrieved from KOHA_CONF by default, provide it from there²
25	has '+conf_file' => (	26	has '+conf_file' => (

Lines 17-22 package Koha::SearchEngine::Zebra::QueryBuilder; Link Here

(-)a/Koha/SearchEngine/Zebra/QueryBuilder.pm (+7 lines)
17	# You should have received a copy of the GNU General Public License	17	# You should have received a copy of the GNU General Public License
18	# along with Koha; if not, see <http://www.gnu.org/licenses>.	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
19		19
		20	use base qw(Class::Accessor);
20	use Modern::Perl;	21	use Modern::Perl;
21	use Moose::Role;	22	use Moose::Role;
22	use C4::Search;	23	use C4::Search;
Lines 28-31 sub build_query { Link Here
28	C4::Search::buildQuery @_;	29	C4::Search::buildQuery @_;
29	}	30	}
30		31
		32	sub build_query_compat {
		33	# Because this passes directly on to C4::Search, we have no trouble being
		34	# compatible.
		35	build_query(@_);
		36	}
		37
31	1;	38	1;

Lines 17-36 package Koha::SearchEngine::Zebra::Search; Link Here

(-)a/Koha/SearchEngine/Zebra/Search.pm (-12 / +30 lines)
17	# You should have received a copy of the GNU General Public License	17	# You should have received a copy of the GNU General Public License
18	# along with Koha; if not, see <http://www.gnu.org/licenses>.	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
19		19
20	use Moose::Role;	20	# I don't think this ever worked right
21	with 'Koha::SearchEngine::SearchRole';	21	#use Moose::Role;
		22	#with 'Koha::SearchEngine::SearchRole';
22		23
23	use Data::SearchEngine::Zebra;	24	use base qw(Class::Accessor);
24	use Data::SearchEngine::Query;	25	# Removed because it doesn't exist/doesn't work.
25	use Koha::SearchEngine::Zebra;	26	#use Data::SearchEngine::Zebra;
26	use Data::Dump qw(dump);	27	#use Data::SearchEngine::Query;
		28	#use Koha::SearchEngine::Zebra;
		29	#use Data::Dump qw(dump);
27		30
28	has searchengine => (	31	use C4::Search; # :(
29	is => 'rw',	32
30	isa => 'Koha::SearchEngine::Zebra',	33	# Broken without the Data:: stuff
31	default => sub { Koha::SearchEngine::Zebra->new },	34	#has searchengine => (
32	lazy => 1	35	# is => 'rw',
33	);	36	# isa => 'Koha::SearchEngine::Zebra',
		37	# default => sub { Koha::SearchEngine::Zebra->new },
		38	# lazy => 1
		39	#);
34		40
35	sub search {	41	sub search {
36	my ($self,$query_string) = @_;	42	my ($self,$query_string) = @_;
Lines 53-58 sub search { Link Here
53	}	59	}
54	}	60	}
55		61
		62	=head2 search_compat
		63
		64	This passes straight through to C4::Search::getRecords.
		65
		66	=cut
		67
		68	sub search_compat {
		69	shift; # get rid of $self
		70
		71	return getRecords(@_);
		72	}
		73
56	sub dosmth {'bou' }	74	sub dosmth {'bou' }
57		75
58	1;	76	1;




DROP TABLE IF EXISTS elasticsearch_mapping;
CREATE TABLE `elasticsearch_mapping` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `mapping` varchar(255) DEFAULT NULL,
  `type` varchar(255) NOT NULL,
  `facet` boolean DEFAULT FALSE,
  `marc21` varchar(255) DEFAULT NULL,
  `unimarc` varchar(255) DEFAULT NULL,
  `normarc` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=126 DEFAULT CHARSET=utf8;



INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('llength',FALSE,'','leader_/1-5',NULL,'leader_/1-5');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('rtype',FALSE,'','leader_/6',NULL,'leader_/6');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bib-level',FALSE,'','leader_/7',NULL,'leader_/7');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('control-number',FALSE,'','001',NULL,'001');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-number',FALSE,'',NULL,'001',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('date-time-last-modified',FALSE,'','005','099d',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('microform-generation',FALSE,'','007_/11',NULL,'007_/11');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('material-type',FALSE,'','007','200b','007');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-00',FALSE,'','007_/1',NULL,'007_/1');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-01',FALSE,'','007_/2',NULL,'007_/2');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-02',FALSE,'','007_/3',NULL,'007_/3');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff7-01-02',FALSE,'','007_/1-2',NULL,'007_/1-2');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('date-entered-on-file',FALSE,'','008_/1-5','099c','008_/1-5');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('pubdate',FALSE,'','008_/7-10','100a_/9-12','008_/7-10');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('pl',FALSE,'','008_/15-17','210a','008_/15-17');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ta',FALSE,'','008_/22','100a_/17','008_/22');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff8-23',FALSE,'','008_/23',NULL,'008_/23');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ff8-29',FALSE,'','008_/29','105a_/8','008_/29');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('lf',FALSE,'','008_/33','105a_/11','008_/33');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bio',FALSE,'','008_/34','105a_/12','008_/34');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ln',FALSE,'','008_/35-37','101a','008_/35-37');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('ctype',FALSE,'','008_/24-27','105a_/4-7','008_/24-27');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('record-source',FALSE,'','008_/39','995c','008_/39');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('lc-cardnumber',FALSE,'','010','995j','010');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('lc-cardnumber',FALSE,'','011',NULL,NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','010',NULL,'010');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','011',NULL,NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bnb-card-number',FALSE,'','015',NULL,'015');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('bgf-number',FALSE,'','015',NULL,'015');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('number-db',FALSE,'','015',NULL,'015');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('number-natl-biblio',FALSE,'','015',NULL,'015');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','015',NULL,'015');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('number-legal-deposit',FALSE,'','017',NULL,NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','017',NULL,NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','018',NULL,NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','020a','010az','020a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('isbn',FALSE,'','020a','010az','020a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('identifier-standard',FALSE,'','022a','011ayz','022a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('issn',FALSE,'','022a','011ayz','022a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','100a','200f','100a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','110a','200g','110a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','111a',NULL,'111a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',TRUE,'string','700a','700a','700a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('author',FALSE,'string','245c','701','245c');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','245a','200a','245a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','246','200c','246');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','247','200d','247');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','490','200e','490a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','505t','200h',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','711t','200i','711t');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','700t','205','700t');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','710t','304a','710t');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','730','327a','730');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','740','327b','740');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','780','327c','780');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','785','327d','785');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','130','327e','130');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','210','327f','210');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','211','327g',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','212','327h',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','214','327i',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','222','328t','222');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string','240','410t','240');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'411t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'412t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'413t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'421t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'422t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'423t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'424t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'425t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'430t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'431t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'432t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'433t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'434t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'435t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'436t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'437t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'440t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'441t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'442t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'443t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'444t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'445t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'446t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'447t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'448t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'451t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'452t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'453t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'454t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'455t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'456t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'461t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'462t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'463t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'464t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'470t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'481t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'482t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('title',FALSE,'string',NULL,'488t',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','600a','600a','600a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','600t','600','600t');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','610a','601','610a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','610t','602','610t');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','611','604','611');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','630n','605','630n');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','630r','606','630r');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650a','607','650a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650b',NULL,'650b');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650c',NULL,'650c');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650d',NULL,'650d');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650v',NULL,'650v');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650x',NULL,'650x');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650y',NULL,'650y');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','650z',NULL,'650z');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','651','608','651');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('subject',TRUE,'string','653a','610','653');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-classification',FALSE,'','952o','995k','952o');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-classification',FALSE,'',NULL,'686',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-number',FALSE,'','999c','001','999c');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('local-number',FALSE,'',NULL,'0909',NULL);
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('itype',TRUE,'string','942c','200b','942c');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('itype',TRUE,'string','952y','995r','952y');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('acqdate',FALSE,'date','952d','9955','952y');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('place',TRUE,'string','260a','210a','260a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('publisher',TRUE,'string','260b','210c','260b');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('copydate',TRUE,'date','260c',NULL,'260c'); -- No copydate for unimarc? Seems strange.
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('homebranch',TRUE,'string','952a','995b','952a');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('holdingbranch',TRUE,'string','952b','995c','952b');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('onloan',FALSE,'boolean','952q','995n','952q');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('itemnumber',FALSE,'number','9529','9959','9529');
INSERT INTO `elasticsearch_mapping` (`mapping`, `facet`, `type`, `marc21`, `unimarc`, `normarc`) VALUES ('issues',FALSE,'sum','952l',NULL,'952l'); -- Apparently not tracked in unimarc




) ENGINE=InnoDB DEFAULT CHARSET=utf8;

--
-- Table structure for table `elasticsearch_mapping`
--

DROP TABLE IF EXISTS `elasticsearch_mapping`;
CREATE TABLE `elasticsearch_mapping` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `mapping` varchar(255) DEFAULT NULL,
  `type` varchar(255) DEFAULT NULL,
  `marc21` varchar(255) DEFAULT NULL,
  `unimarc` varchar(255) DEFAULT NULL,
  `normarc` varchar(255) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8;

--
-- Table structure for table `ethnicity`
--


Lines 112-115 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+1 lines)
112	choices:	112	choices:
113	Solr: Solr	113	Solr: Solr
114	Zebra: Zebra	114	Zebra: Zebra
		115	Elasticsearch: Elasticsearch
115	- is the search engine used.	116	- is the search engine used.





<script type="text/javascript">
  $(document).ready(function() {
//    $('#bookbag_form').find("input").hide();
    $('#sort_by').change(function() {
        $('#bookbag_form').submit();
    });

  </div>
  <div class="searchresults">
    <form action="/cgi-bin/koha/opac-search.pl" method="get" name="bookbag_form" id="bookbag_form">
        [%# IF (browse) %]
            <label for="browse_field">Browse: </label>
            <select name="browse_field" id="browse_field">
                <option value="title">Title</option>
                <option value="author">Author</option>
                <option value="callnumber">Call Number</option>
                <option value="subject">Subject</option>
                <option value="isbn">ISBN</option>
                <option value="issn">ISSN</option>
            </select>
            <input type="hidden" name="type" value="browse" />
            <br />
            <label for="search_field">Query:</label>
            <input type="text" name="q" style="display:initial;" />
            <input type="submit" value="Browse" style="display:initial;" />
        [%# END %]
      <!-- TABLE RESULTS START -->
      <table>
        <thead>

        </thead>
        <!-- Actual Search Results -->
        <tbody>
        [% USE Dumper %]
          [% FOREACH SEARCH_RESULT IN SEARCH_RESULTS %]
           [% result =SEARCH_RESULT.item('_source') %]
            <tr>
              <td>
                <input type="checkbox" id="bib[% result.biblionumber %]" name="biblionumber" value="[% result.biblionumber %]" /> <label for="bib[% result.biblionumber %]"></label>
              </td>
              <td>
                <a class="title" href="/cgi-bin/koha/opac-detail.pl?biblionumber=[% result.biblionumber |url %]" title="View details for this title">[% result.title |html %]</a>
                by <a href="/cgi-bin/koha/opac-search.pl?q=author:[% result.author |url %]" title="Search for works by this author" class="author">[% result.author %]</a>
              </td>
            </tr>
          [% END %]




#!/usr/bin/perl

# This inserts records from a Koha database into elastic search

# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

=head1 NAME

rebuild_elastic_search.pl - inserts records from a Koha database into Elasticsearch

=head1 SYNOPSIS

B<rebuild_elastic_search.pl>
[B<-c|--commit>=C<count>]
[B<-v|--verbose>]
[B<-h|--help>]
[B<--man>]

=head1 DESCRIPTION

=head1 OPTIONS

=over

=item B<-c|--commit>=C<count>

Specify how many records will be batched up before they're added to Elasticsearch.
Higher should be faster, but will cause more RAM usage. Default is 100.

=item B<-d|--delete>

Delete the index and recreate it before indexing.

=item B<-b|--biblionumber>

Only index the supplied biblionumber, mostly for testing purposes. May be
repeated.

=item B<-v|--verbose>

By default, this program only emits warnings and errors. This makes it talk
more. Add more to make it even more wordy, in particular when debugging.

=item B<-h|--help>

Help!

=item B<--man>

Full documentation.

=cut

use autodie;
use Getopt::Long;
use Koha::Biblio;
use Koha::ElasticSearch::Indexer;
use MARC::Field;
use MARC::Record;
use Modern::Perl;
use Pod::Usage;

use Data::Dumper; # TODO remove

my $verbose = 0;
my $commit = 100;
my ($delete, $help, $man);
my (@biblionumbers);

GetOptions(
    'c|commit=i'       => \$commit,
    'd|delete'         => \$delete,
    'b|biblionumber=i' => \@biblionumbers,
    'v|verbose!'       => \$verbose,
    'h|help'           => \$help,
    'man'              => \$man,
);

pod2usage(1) if $help;
pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;

my $next;
if (@biblionumbers) {
    $next = sub {
        my $r = shift @biblionumbers;
        return () unless defined $r;
        return ($r, Koha::Biblio->get_marc_biblio($r, item_data => 1));
    };
} else {
    my $records = Koha::Biblio->get_all_biblios_iterator();
    $next = sub {
        $records->next();
    }
}
my $indexer = Koha::ElasticSearch::Indexer->new({index => 'biblios' });
if ($delete) {
    # We know it's safe to not recreate the indexer because update_index
    # hasn't been called yet.
    $indexer->delete_index();
}

my $count = 0;
my $commit_count = $commit;
my (@bibnums_buffer, @commit_buffer);
while (scalar(my ($bibnum, $rec) = $next->())) {
    _log(1,"$bibnum\n");
    $count++;

    push @bibnums_buffer, $bibnum;
    push @commit_buffer, $rec;
    if (!(--$commit_count)) {
        _log(2, "Committing...\n");
        $indexer->update_index(\@bibnums_buffer, \@commit_buffer);
        $commit_count = $commit;
        @bibnums_buffer = ();
        @commit_buffer = ();
    }
}
# There are probably uncommitted records
$indexer->update_index(\@bibnums_buffer, \@commit_buffer);
_log(1, "$count records indexed.\n");

# Output progress information.
#
#   _log($level, $msg);
#
# Will output $msg if the verbosity setting is set to $level or more. Will
# not include a trailing newline.
sub _log {
    my ($level, $msg) = @_;

    print $msg if ($verbose <= $level);
}

Line 0 Link Here

(-)a/myfix.txt (+3 lines)
	1	marc_map( '245a','title' );
	2	marc_map( '100a','author' );
	3	marc_map( '999c','biblionumber' );




#!/usr/bin/perl

# Copyright 2013 Catalyst
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use C4::Context;
use CGI;
use C4::Auth;
use C4::Koha;
use C4::Output;

# TODO this should use the moose thing that auto-picks.
use Koha::SearchEngine::Elasticsearch::QueryBuilder;
use Koha::ElasticSearch::Search;

my $cgi = new CGI;

my $template_name;
my $template_type = "basic";
if ( $cgi->param("idx") or $cgi->param("q") ) {
    $template_name = 'search/results.tt';
}
else {
    $template_name = 'search/advsearch.tt';
    $template_type = 'advsearch';
}

# load the template
my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
    {
        template_name   => $template_name,
        query           => $cgi,
        type            => "opac",
        authnotrequired => 1,
    }
);
my %template_params;
my $format = $cgi->param("format") || 'html';

# load the Type stuff
my $itemtypes = GetItemTypes;

my $page = $cgi->param("page") || 1;
my $count =
     $cgi->param('count')
  || C4::Context->preference('OPACnumSearchResults')
  || 20;
my $q = $cgi->param("q");

my $searcher = Koha::ElasticSearch::Search->new();
my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
my $query;
if ($cgi->param('type') eq 'browse') {
    $query = $builder->build_browse_query($cgi->param('browse_field') || undef, $q );
    $template_params{browse} = 1;
} else {
    $query = $builder->build_query($q);
}
my $results = $searcher->search( $query, $page, $count );
#my $results = $searcher->search( { "match_phrase_prefix" => { "title" => "the" } } );

# This is temporary, but will do the job for now.
my @hits;
$results->each(sub {
        push @hits, { _source => @_[0] };
    });
# Make a list of the page numbers
my @pages = map { { page => $_, current => ($_ == ( $page || 1)) } } 1 .. int($results->total / $count);
my $max_page = int($results->total / $count);
# Pager template params
$template->param(
    SEARCH_RESULTS  => \@hits,
    PAGE_NUMBERS    => \@pages,
    total           => $results->total,
    previous_page   => ( $page > 1 ? $page - 1 : undef ),
    next_page       => ( $page < $max_page ? $page + 1 : undef ),
    follower_params => [
        { var => 'type',  val => $cgi->param('type') },
        { var => 'q',     val => $q },
        { var => 'count', val => $count },
    ],
    %template_params,
);

my $content_type = ( $format eq 'rss' or $format eq 'atom' ) ? $format : 'html';
output_with_http_headers $cgi, $cookie, $template->output, $content_type;




# to perform, etc.
## load Koha modules
use C4::Context;
use C4::Search;

use Data::Dumper; # TODO remove
if ( $searchengine =~ /^Solr$/ ) {
    warn "We use Solr";
    require 'opac/search.pl';
    exit;
} elsif ( $searchengine =~ /^Zebra$/ ) {

use Koha::SearchEngine::Elasticsearch::QueryBuilder;
use Koha::ElasticSearch::Search;
use Koha::SearchEngine::Zebra::QueryBuilder;
use Koha::SearchEngine::Zebra::Search;

my $searchengine = C4::Context->preference("SearchEngine");
my ($builder, $searcher);
#$searchengine = 'Zebra'; # XXX
for ( $searchengine ) {
    when ( /^Solr$/ ) {
        warn "We use Solr";
        require 'opac/search.pl';
        exit;
    }
    when ( /^Zebra$/ ) {
        $builder=Koha::SearchEngine::Zebra::QueryBuilder->new();
        $searcher=Koha::SearchEngine::Zebra::Search->new();
    }
    when (/^Elasticsearch$/) {
        # Should use the base QueryBuilder, but I don't have it wired up
        # for moose yet.
        $builder=Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
#        $builder=Koha::SearchEngine::Zebra::QueryBuilder->new();
        $searcher=Koha::ElasticSearch::Search->new({index => 'biblios'});
    }
}

use C4::Output;

my @results;

## I. BUILD THE QUERY
( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type) = $builder->build_query_compat(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang);
#die Dumper( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type);

sub _input_cgi_parse {
    my @elements;

    $pasarParams .= '&amp;simple_query=' . $simple_query;
    $pasarParams .= '&amp;query_type=' . $query_type if ($query_type);
    eval {
        ($error, $results_hashref, $facets) = $searcher->search_compat($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan,1);
    };
}

# This sorts the facets into alphabetical order
if ($facets && @$facets) {
    foreach my $f (@$facets) {
        $f->{facets} = [ sort { uc($a->{facet_title_value}) cmp uc($b->{facet_title_value}) } @{ $f->{facets} } ];
    }




#!/usr/bin/perl
#
# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Test::More tests => 8;

BEGIN {
    use_ok('Koha::ItemType');
}

my $data = {
    itemtype       => 'CODE',
    description    => 'description',
    rentalcharge   => 'rentalcharge',
    imageurl       => 'imageurl',
    summary        => 'summary',
    checkinmsg     => 'checkinmsg',
    checkinmsgtype => 'checkinmsgtype',
};

my $type = Koha::ItemType->new($data);

is( $type->code,           'CODE',           'itemtype/code' );
is( $type->description,    'description',    'description' );
is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
is( $type->imageurl,       'imageurl',       'imageurl' );
is( $type->summary,        'summary',        'summary' );
is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );

Line 0 Link Here

(-)a/t/Koha_ElasticSearch.t (+23 lines)
	1	#
	2	#===============================================================================
	3	#
	4	# FILE: Koha_ElasticSearch.t
	5	#
	6	# DESCRIPTION:
	7	#
	8	# FILES: ---
	9	# BUGS: ---
	10	# NOTES: ---
	11	# AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
	12	# ORGANIZATION: Koha Development Team
	13	# VERSION: 1.0
	14	# CREATED: 09/12/13 08:56:44
	15	# REVISION: ---
	16	#===============================================================================
	17
	18	use strict;
	19	use warnings;
	20
	21	use Test::More tests => 1; # last test to print
	22
	23	use_ok('Koha::ElasticSearch');




#
#===============================================================================
#
#         FILE: Koha_ElasticSearch_Indexer.t
#
#  DESCRIPTION:
#
#        FILES: ---
#         BUGS: ---
#        NOTES: ---
#       AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
# ORGANIZATION: Koha Development Team
#      VERSION: 1.0
#      CREATED: 09/12/13 08:57:25
#     REVISION: ---
#===============================================================================

use strict;
use warnings;

use Test::More tests => 5;    # last test to print
use MARC::Record;

use_ok('Koha::ElasticSearch::Indexer');

my $indexer;
ok(
    my $indexer = Koha::ElasticSearch::Indexer->new(
        {
            'nodes' => ['localhost:9200'],
            'index' => 'mydb'
        }
    ),
    'Creating new indexer object'
);

my $marc_record = MARC::Record->new();
my $field = MARC::Field->new( '001', '1234567' );
$marc_record->append_fields($field);
$field = MARC::Field->new( '020', '', '', 'a' => '1234567890123' );
$marc_record->append_fields($field);
$field = MARC::Field->new( '245', '', '', 'a' => 'Title' );
$marc_record->append_fields($field);

my $records = [$marc_record];
ok( my $converted = $indexer->convert_marc_to_json($records),
    'Convert some records' );

is( $converted->count, 1, 'One converted record' );

ok( $indexer->update_index($records), 'Update Index' );




#
#===============================================================================
#
#         FILE: Koha_ElasticSearch_Search.t
#
#  DESCRIPTION:
#
#        FILES: ---
#         BUGS: ---
#        NOTES: ---
#       AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
# ORGANIZATION: Koha Development Team
#      VERSION: 1.0
#      CREATED: 09/12/13 09:43:29
#     REVISION: ---
#===============================================================================

use strict;
use warnings;

use Test::More tests => 5;    # last test to print

use_ok('Koha::ElasticSearch::Search');

ok(
    my $searcher = Koha::ElasticSearch::Search->new(
        { 'nodes' => ['localhost:9200'], 'index' => 'mydb' }
    ),
    'Creating a Koha::ElasticSearch::Search object'
);

is( $searcher->index, 'mydb', 'Testing basic accessor' );

ok( $searcher->connect, 'Connect to ElasticSearch server' );
ok( my $results = $searcher->search( { record => 'easy' } ), 'Do a search ' );

ok( my $marcresults = $searcher->marc_search( { record => 'Fish' } ),
    'Do a marc search' );




#!/usr/bin/perl
#
# Copyright 2014 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

# XXX This doesn't work because I need to figure out how to do transactions
# in a test-case with DBIx::Class

use Modern::Perl;

use Test::More tests => 8;
use Data::Dumper;

BEGIN {
    use_ok('Koha::ItemTypes');
}

my $dbh = C4::Context->dbh;

# Start transaction
$dbh->{AutoCommit} = 0;
$dbh->{RaiseError} = 1;

my $prep = $dbh->prepare('INSERT INTO itemtypes (itemtype, description, rentalcharge, imageurl, summary, checkinmsg, checkinmsgtype) VALUES (?,?,?,?,?,?,?)');
$prep->execute('type1', 'description', 'rentalcharge', 'imageurl', 'summary', 'checkinmsg', 'checkinmsgtype');
$prep->execute('type2', 'description', 'rentalcharge', 'imageurl', 'summary', 'checkinmsg', 'checkinmsgtype');

my $itypes = Koha::ItemTypes->new();

my @types = $itypes->get_itemtype('type1', 'type2');

die Dumper(\@types);
my $type = $types[0];
ok(defined($type), 'first result');
is( $type->code,           'type1',           'itemtype/code' );
is( $type->description,    'description',    'description' );
is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
is( $type->imageurl,       'imageurl',       'imageurl' );
is( $type->summary,        'summary',        'summary' );
is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );

$type = $types[1];
ok(defined($type), 'second result');
is( $type->code,           'type2',           'itemtype/code' );
is( $type->description,    'description',    'description' );
is( $type->rentalcharge,   'rentalcharge',   'rentalcharge' );
is( $type->imageurl,       'imageurl',       'imageurl' );
is( $type->summary,        'summary',        'summary' );
is( $type->checkinmsg,     'checkinmsg',     'checkinmsg' );
is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );

Return to bug 12478

Lines 931-936 CREATE TABLE `deleteditems` ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+15 lines)
931	) ENGINE=InnoDB DEFAULT CHARSET=utf8;	931	) ENGINE=InnoDB DEFAULT CHARSET=utf8;
932		932
933	--	933	--
		934	-- Table structure for table `elasticsearch_mapping`
		935	--
		936
		937	DROP TABLE IF EXISTS `elasticsearch_mapping`;
		938	CREATE TABLE `elasticsearch_mapping` (
		939	`id` int(11) NOT NULL AUTO_INCREMENT,
		940	`mapping` varchar(255) DEFAULT NULL,
		941	`type` varchar(255) DEFAULT NULL,
		942	`marc21` varchar(255) DEFAULT NULL,
		943	`unimarc` varchar(255) DEFAULT NULL,
		944	`normarc` varchar(255) DEFAULT NULL,
		945	PRIMARY KEY (`id`)
		946	) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8;
		947
		948	--
934	-- Table structure for table `ethnicity`	949	-- Table structure for table `ethnicity`
935	--	950	--
936		951

Line 0 Link Here

(-)a/Koha/Biblio.pm (+105 lines)
		1	package Koha::Biblio;
		2
		3	# This contains functions to do with managing biblio records.
		4
		5	# Copyright 2014 Catalyst IT
		6	#
		7	# This file is part of Koha.
		8	#
		9	# Koha is free software; you can redistribute it and/or modify it under the
		10	# terms of the GNU General Public License as published by the Free Software
		11	# Foundation; either version 3 of the License, or (at your option) any later
		12	# version.
		13	#
		14	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		15	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		16	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		17	#
		18	# You should have received a copy of the GNU General Public License along
		19	# with Koha; if not, write to the Free Software Foundation, Inc.,
		20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		21
		22	=head1 NAME
		23
		24	Koha::Biblio - contains fundamental biblio-related functions
		25
		26	=head1 DESCRIPTION
		27
		28	This contains functions for normal operations on biblio records.
		29
		30	Note: really, C4::Biblio does the main functions, but the Koha namespace is
		31	the new thing that should be used.
		32
		33	=cut
		34
		35	use C4::Biblio; # EmbedItemsInMarcBiblio
		36	use Koha::Biblio::Iterator;
		37	use Koha::Database;
		38	use Modern::Perl;
		39
		40	use base qw(Class::Accessor);
		41
		42	__PACKAGE__->mk_accessors(qw());
		43
		44	=head1 FUNCTIONS
		45
		46	=head2 get_all_biblios_iterator
		47
		48	my $it = get_all_biblios_iterator();
		49
		50	This will provide an iterator object that will, one by one, provide the
		51	MARC::Record of each biblio. This will include the item data.
		52
		53	The iterator is a Koha::Biblio::Iterator object.
		54
		55	=cut
		56
		57	sub get_all_biblios_iterator {
		58	my $database = Koha::Database->new();
		59	my $schema = $database->schema();
		60	my $rs =
		61	$schema->resultset('Biblioitem')->search( { marc => { '!=', undef } },
		62	{ columns => [qw/ biblionumber marc /] } );
		63	return Koha::Biblio::Iterator->new($rs, items => 1);
		64	}
65
66	=head2 get_marc_biblio
67
68	my $marc = get_marc_biblio($bibnum, %options);
69
70	This fetches the MARC::Record for the given biblio number. Nothing is returned
71	if the biblionumber couldn't be found (or it somehow has no MARC data.)
72
73	Options are:
74
75	=over 4
76
77	=item item_data
78
79	If set to true, item data is embedded in the record. Default is to not do this.
80
81	=back
82
83	=cut
84
85	sub get_marc_biblio {
86	my ($class,$bibnum, %options) = @_;
87
88	my $database = Koha::Database->new();
89	my $schema = $database->schema();
90	my $rs =
91	$schema->resultset('Biblioitem')
92	->search( { marc => { '!=', undef }, biblionumber => $bibnum },
93	{ columns => [qw/ marc /] } );
94
95	my $row = $rs->next();
96	return unless $row;
97	my $marc = MARC::Record->new_from_usmarc($row->marc);
98
99	# TODO implement this in this module
100	C4::Biblio::EmbedItemsInMarcBiblio($marc, $bibnum) if $options{item_data};
101
102	return $marc;
103	}
104
105	1;

Line 0 Link Here

(-)a/Koha/Biblio/Iterator.pm (+126 lines)
		1	package Koha::Biblio::Iterator;
		2
		3	# This contains an iterator over biblio records
		4
		5	# Copyright 2014 Catalyst IT
		6	#
		7	# This file is part of Koha.
		8	#
		9	# Koha is free software; you can redistribute it and/or modify it under the
		10	# terms of the GNU General Public License as published by the Free Software
		11	# Foundation; either version 3 of the License, or (at your option) any later
		12	# version.
		13	#
		14	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		15	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		16	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		17	#
		18	# You should have received a copy of the GNU General Public License along
		19	# with Koha; if not, write to the Free Software Foundation, Inc.,
		20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		21
		22	=head1 NAME
		23
		24	Koha::Biblio::Iterator - iterates over biblios provided by a DBIx::Class::ResultSet
		25
		26	=head1 DESCRIPTION
		27
		28	This provides an iterator that gives the MARC::Record of each biblio that's
		29	returned by a L<DBIx::Class::ResultSet> that provides a C<biblionumber>, and
		30	C<marc> or C<marcxml> column from the biblioitems table.
		31
		32	=head1 SYNOPSIS
		33
		34	use Koha::Biblio::Iterator;
		35	my $rs = $schema->resultset('biblioitems');
		36	my $iterator = Koha::Biblio::Iterator->new($rs);
		37	while (my $record = $iterator->next()) {
		38	// do something with $record
		39	}
		40
		41	=head1 METHODS
		42
		43	=cut
		44
		45	use C4::Biblio; # :( - for EmbedItemsInMarcBiblio
		46
		47	use Carp;
		48	use MARC::Record;
		49	use MARC::File::XML;
		50	use Modern::Perl;
		51
		52	=head2 new
		53
		54	my $it = new($sth, option => $value, ...);
		55
		56	Takes a ResultSet to iterate over, and gives you an iterator on it. Optional
		57	options may be specified.
		58
		59	=head3 Options
		60
		61	=over 4
		62
		63	=item items
		64
65	Set to true to include item data in the resulting MARC record.
66
67	=back
68
69	=cut
70
71	sub new {
72	my ( $class, $rs, %options ) = @_;
73
74	bless {
75	rs => $rs,
76	%options,
77	}, $class;
78	}
79
80	=head2 next()
81
82	In a scalar context, provides the next MARC::Record from the ResultSet, or
83	C<undef> if there are no more.
84
85	In a list context it will provide ($biblionumber, $record).
86
87	=cut
88
89	sub next {
90	my ($self) = @_;
91
92	my $marc;
93	my $row = $self->{rs}->next();
94	return if !$row;
95	if ( $row->marc ) {
96	$marc = MARC::Record->new_from_usmarc( $row->marc );
97	}
98	elsif ( $row->marcxml ) {
99	$marc = MARC::Record->new_from_xml( $row->marcxml );
100	}
101	else {
102	confess "No marc or marcxml column returned in the request.";
103	}
104
105	my $bibnum;
106	if ( $self->{items} ) {
107	$bibnum = $row->get_column('biblionumber');
108	confess "No biblionumber column returned in the request."
109	if ( !defined($bibnum) );
110
111	# TODO this should really be in Koha::Biblio or something similar.
112	C4::Biblio::EmbedItemsInMarcBiblio( $marc, $bibnum );
113	}
114
115	if (wantarray) {
116	$bibnum //= $row->get_column('biblionumber');
117	confess "No biblionumber column returned in the request."
118	if ( !defined($bibnum) );
119	return ( $bibnum, $marc );
120	}
121	else {
122	return $marc;
123	}
124	}
125
126	1;

Lines 40-45 use base qw(Class::Accessor); Link Here

(-)a/Koha/Database.pm (-60 / +7 lines)
40		40
41	__PACKAGE__->mk_accessors(qw( ));	41	__PACKAGE__->mk_accessors(qw( ));
42		42
		43	our $schema; # the schema is a singleton
		44
43	# _new_schema	45	# _new_schema
44	# Internal helper function (not a method!). This creates a new	46	# Internal helper function (not a method!). This creates a new
45	# database connection from the data given in the current context, and	47	# database connection from the data given in the current context, and
Lines 64-85 creates one, and connects to the database. Link Here
64		66
65	This database handle is cached for future use: if you call	67	This database handle is cached for future use: if you call
66	C<$database-E<gt>schema> twice, you will get the same handle both	68	C<$database-E<gt>schema> twice, you will get the same handle both
67	times. If you need a second database handle, use C<&new_schema> and	69	times. If you need a second database handle, use C<&new_schema>.
68	possibly C<&set_schema>.
69		70
70	=cut	71	=cut
71		72
72	sub schema {	73	sub schema {
73	my $self = shift;	74	my $self = shift;
74	my $sth;	75	my $sth;
75	if ( defined( $self->{"schema"} ) && $self->{"schema"}->storage->connected() ) {	76	if ( defined( $schema ) && $schema->storage->connected() ) {
76	return $self->{"schema"};	77	return $schema;
77	}	78	}
78		79
79	# No database handle or it died . Create one.	80	# No database handle or it died . Create one.
80	$self->{"schema"} = &_new_schema();	81	$schema = &_new_schema();
81		82
82	return $self->{"schema"};	83	return $schema;
83	}	84	}
84		85
85	=head2 new_schema	86	=head2 new_schema
Lines 102-161 sub new_schema { Link Here
102	return &_new_schema();	103	return &_new_schema();
103	}	104	}
104		105
105	=head2 set_schema
106
107	$my_schema = $database->new_schema;
108	$database->set_schema($my_schema);
109	...
110	$database->restore_schema;
111
112	C<&set_schema> and C<&restore_schema> work in a manner analogous to
113	C<&set_context> and C<&restore_context>.
114
115	C<&set_schema> saves the current database handle on a stack, then sets
116	the current database handle to C<$my_schema>.
117
118	C<$my_schema> is assumed to be a good database handle.
119
120	=cut
121
122	sub set_schema {
123	my $self = shift;
124	my $new_schema = shift;
125
126	# Save the current database handle on the handle stack.
127	# We assume that $new_schema is all good: if the caller wants to
128	# screw himself by passing an invalid handle, that's fine by
129	# us.
130	push @{ $self->{"schema_stack"} }, $self->{"schema"};
131	$self->{"schema"} = $new_schema;
132	}
133
134	=head2 restore_schema
135
136	$database->restore_schema;
137
138	Restores the database handle saved by an earlier call to
139	C<$database-E<gt>set_schema>.
140
141	=cut
142
143	sub restore_schema {
144	my $self = shift;
145
146	if ( $#{ $self->{"schema_stack"} } < 0 ) {
147
148	# Stack underflow
149	die "SCHEMA stack underflow";
150	}
151
152	# Pop the old database handle and set it.
153	$self->{"schema"} = pop @{ $self->{"schema_stack"} };
154
155	# FIXME - If it is determined that restore_context should
156	# return something, then this function should, too.
157	}
158
159	=head2 EXPORT	106	=head2 EXPORT
160		107
161	None by default.	108	None by default.

Line 0 Link Here

(-)a/Koha/ElasticSearch.pm (+323 lines)
		1	package Koha::ElasticSearch;
		2
		3	# Copyright 2013 Catalyst IT
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use base qw(Class::Accessor);
		21
		22	use C4::Context;
		23	use Carp;
		24	use Elasticsearch;
		25	use Koha::Database;
		26	use Modern::Perl;
		27
		28	use Data::Dumper; # TODO remove
		29
		30	__PACKAGE__->mk_ro_accessors(qw( index ));
		31
		32	=head1 NAME
		33
		34	Koha::ElasticSearch - Base module for things using elasticsearch
		35
		36	=head1 ACCESSORS
		37
		38	=over 4
		39
		40	=item index
		41
		42	The name of the index to use, generally 'biblios' or 'authorities'.
		43
		44	=back
		45
		46	=head1 FUNCTIONS
		47
		48	=cut
		49
		50	sub new {
		51	my $class = shift @_;
		52	my $self = $class->SUPER::new(@_);
		53	# Check for a valid index
		54	croak('No index name provided') unless $self->index;
		55	return $self;
		56	}
		57
		58	=head2 get_elasticsearch_params
		59
		60	my $params = $self->get_elasticsearch_params();
		61
		62	This provides a hashref that contains the parameters for connecting to the
		63	ElasicSearch servers, in the form:
		64
65	{
66	'servers' => ['127.0.0.1:9200', 'anotherserver:9200'],
67	'index_name' => 'koha_instance',
68	}
69
70	This is configured by the following in the C<config> block in koha-conf.xml:
71
72	<elasticsearch>
73	<server>127.0.0.1:9200</server>
74	<server>anotherserver:9200</server>
75	<index_name>koha_instance</index_name>
76	</elasticsearch>
77
78	=cut
79
80	sub get_elasticsearch_params {
81	my ($self) = @_;
82
83	# Copy the hash so that we're not modifying the original
84	my $es = { %{ C4::Context->config('elasticsearch') } };
85	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$es );
86
87	# Helpfully, the multiple server lines end up in an array for us anyway
88	# if there are multiple ones, but not if there's only one.
89	my $server = $es->{server};
90	delete $es->{server};
91	if ( ref($server) eq 'ARRAY' ) {
92
93	# store it called 'servers'
94	$es->{servers} = $server;
95	}
96	elsif ($server) {
97	$es->{servers} = [$server];
98	}
99	else {
100	die "No elasticsearch servers were specified in koha-conf.xml.\n";
101	}
102	die "No elasticserver index_name was specified in koha-conf.xml.\n"
103	if ( !$es->{index_name} );
104	# Append the name of this particular index to our namespace
105	$es->{index_name} .= '_' . $self->index;
106	return $es;
107	}
108
109	=head2 get_elasticsearch_settings
110
111	my $settings = $self->get_elasticsearch_settings();
112
113	This provides the settings provided to elasticsearch when an index is created.
114	These can do things like define tokenisation methods.
115
116	A hashref containing the settings is returned.
117
118	=cut
119
120	sub get_elasticsearch_settings {
121	my ($self) = @_;
122
123	# Ultimately this should come from a file or something, and not be
124	# hardcoded.
125	my $settings = {
126	index => {
127	analysis => {
128	analyzer => {
129	analyser_phrase => {
130	tokenizer => 'keyword',
131	filter => 'lowercase',
132	},
133	analyser_standard => {
134	tokenizer => 'standard',
135	filter => 'lowercase',
136	}
137	}
138	}
139	}
140	};
141	return $settings;
142	}
143
144	=head2 get_elasticsearch_mappings
145
146	my $mappings = $self->get_elasticsearch_mappings();
147
148	This provides the mappings that get passed to elasticsearch when an index is
149	created.
150
151	=cut
152
153	sub get_elasticsearch_mappings {
154	my ($self) = @_;
155
156	my $mappings = {
157	data => {
158	properties => {
159	record => {
160	store => "yes",
161	include_in_all => "false",
162	type => "string",
163	},
164	}
165	}
166	};
167	$self->_foreach_mapping(
168	sub {
169	my ( undef, $name, $type, $facet ) = @_;
170
171	# TODO if this gets any sort of complexity to it, it should
172	# be broken out into its own function.
173
174	# TODO be aware of date formats, but this requires pre-parsing
175	# as ES will simply reject anything with an invalid date.
176	my $es_type =
177	$type eq 'boolean'
178	? 'boolean'
179	: 'string';
180	$mappings->{data}{properties}{$name} = {
181	search_analyzer => "analyser_standard",
182	index_analyzer => "analyser_standard",
183	type => $es_type,
184	fields => {
185	phrase => {
186	search_analyzer => "analyser_phrase",
187	index_analyzer => "analyser_phrase",
188	type => "string"
189	},
190	},
191	};
192	$mappings->{data}{properties}{$name}{null_value} = 0
193	if $type eq 'boolean';
194	if ($facet) {
195	$mappings->{data}{properties}{ $name . '__facet' } = {
196	type => "string",
197	index => "not_analyzed",
198	};
199	}
200	}
201	);
202	return $mappings;
203	}
204
205	# Provides the rules for data conversion.
206	sub get_fixer_rules {
207	my ($self) = @_;
208
209	my $marcflavour = lc C4::Context->preference('marcflavour');
210	my @rules;
211	$self->_foreach_mapping(
212	sub {
213	my ( undef, $name, $type, $facet, $marcs ) = @_;
214	my $field = $marcs->{$marcflavour};
215	return unless defined $marcs->{$marcflavour};
216	my $options = '';
217
218	# There's a bug when using 'split' with something that
219	# selects a range
220	# The split makes everything into nested arrays, but that's not
221	# really a big deal, ES doesn't mind.
222	$options = '-split => 1' unless $field =~ m\|_/\| \|\| $type eq 'sum';
223	push @rules, "marc_map('$field','${name}', $options)";
224	if ($facet) {
225	push @rules, "marc_map('$field','${name}__facet', $options)";
226	}
227	if ( $type eq 'boolean' ) {
228
229	# boolean gets special handling, basically if it doesn't exist,
230	# it's added and set to false. Otherwise we can't query it.
231	push @rules,
232	"unless exists('$name') add_field('$name', 0) end";
233	}
234	if ($type eq 'sum' ) {
235	push @rules, "sum('$name')";
236	}
237	}
238	);
239
240	return \@rules;
241	}
242
243	=head2 _foreach_mapping
244
245	$self->_foreach_mapping(
246	sub {
247	my ( $id, $name, $type, $facet, $marcs ) = @_;
248	my $marc = $marcs->{marc21};
249	}
250	);
251
252	This allows you to apply a function to each entry in the elasticsearch mappings
253	table, in order to build the mappings for whatever is needed.
254
255	In the provided function, the files are:
256
257	=over 4
258
259	=item C<$id>
260
261	An ID number, corresponding to the entry in the database.
262
263	=item C<$name>
264
265	The field name for elasticsearch (corresponds to the 'mapping' column in the
266	database.
267
268	=item C<$type>
269
270	The type for this value, e.g. 'string'.
271
272	=item C<$facet>
273
274	True if this value should be facetised. This only really makes sense if the
275	field is understood by the facet processing code anyway.
276
277	=item C<$marc>
278
279	A hashref containing the MARC field specifiers for each MARC type. It's quite
280	possible for this to be undefined if there is otherwise an entry in a
281	different MARC form.
282
283	=back
284
285	=cut
286
287	sub _foreach_mapping {
288	my ( $self, $sub ) = @_;
289
290	# TODO use a caching framework here
291	my $database = Koha::Database->new();
292	my $schema = $database->schema();
293	my $rs = $schema->resultset('ElasticsearchMapping')->search();
294	for my $row ( $rs->all ) {
295	$sub->(
296	$row->id,
297	$row->mapping,
298	$row->type,
299	$row->facet,
300	{
301	marc21 => $row->marc21,
302	unimarc => $row->unimarc,
303	normarc => $row->normarc
304	}
305	);
306	}
307	}
308
309	1;
310
311	__END__
312
313	=head1 AUTHOR
314
315	=over 4
316
317	=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
318
319	=item Robin Sheat C<< <robin@catalyst.net.nz> >>
320
321	=back
322
323	=cut

Line 0 Link Here

(-)a/Koha/ElasticSearch/Indexer.pm (+155 lines)
		1	package Koha::ElasticSearch::Indexer;
		2
		3	# Copyright 2013 Catalyst IT
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Carp;
		21	use Modern::Perl;
		22	use base qw(Koha::ElasticSearch);
		23	use Data::Dumper;
		24
		25	# For now just marc, but we can do anything here really
		26	use Catmandu::Importer::MARC;
		27	use Catmandu::Store::ElasticSearch;
		28
		29	Koha::ElasticSearch::Indexer->mk_accessors(qw( store ));
		30
		31	=head1 NAME
		32
		33	Koha::ElasticSearch::Indexer - handles adding new records to the index
		34
		35	=head1 SYNOPSIS
		36
		37	my $indexer = Koha::ElasticSearch::Indexer->new({ index => 'biblios' });
		38	$indexer->delete_index();
		39	$indexer->update_index(\@biblionumbers, \@records);
		40
		41	=head1 FUNCTIONS
		42
		43	=cut
		44
		45	=head2 $indexer->update_index($biblionums, $records);
		46
		47	C<$biblionums> is an arrayref containing the biblionumbers for the records.
		48
		49	C<$records> is an arrayref containing the L<MARC::Record>s themselves.
		50
		51	The values in the arrays must match up, and the 999$c value in the MARC record
		52	will be rewritten using the values in C<$biblionums> to ensure they are correct.
		53	If C<$biblionums> is C<undef>, this won't happen, but you should be sure that
		54	999$c is correct on your own then.
		55
		56	Note that this will modify the original record if C<$biblionums> is supplied.
		57	If that's a problem, clone them first.
		58
		59	=cut
		60
		61	sub update_index {
		62	my ($self, $biblionums, $records) = @_;
		63
		64	if ($biblionums) {
65	$self->_sanitise_records($biblionums, $records);
66	}
67
68	my $from = $self->_convert_marc_to_json($records);
69	if ( !$self->store ) {
70	my $params = $self->get_elasticsearch_params();
71	$self->store(
72	Catmandu::Store::ElasticSearch->new(
73	%$params,
74	index_settings => $self->get_elasticsearch_settings(),
75	index_mappings => $self->get_elasticsearch_mappings(),
76	#trace_calls => 1,
77	)
78	);
79	}
80	$self->store->bag->add_many($from);
81	$self->store->bag->commit;
82	return 1;
83	}
84
85	=head2 $indexer->delete_index();
86
87	Deletes the index from the elasticsearch server. Calling C<update_index>
88	after this will recreate it again.
89
90	=cut
91
92	sub delete_index {
93	my ($self) = @_;
94
95	if (!$self->store) {
96	# If this index doesn't exist, this will create it. Then it'll be
97	# deleted. That's not the end of the world however.
98	my $params = $self->get_elasticsearch_params();
99	$self->store(
100	Catmandu::Store::ElasticSearch->new(
101	%$params,
102	index_settings => $self->get_elasticsearch_settings(),
103	index_mappings => $self->get_elasticsearch_mappings(),
104	#trace_calls => 1,
105	)
106	);
107	}
108	$self->store->drop();
109	$self->store(undef);
110	}
111
112	sub _sanitise_records {
113	my ($self, $biblionums, $records) = @_;
114
115	confess "Unequal number of values in \$biblionums and \$records." if (@$biblionums != @$records);
116
117	my $c = @$biblionums;
118	for (my $i=0; $i<$c; $i++) {
119	my $bibnum = $biblionums->[$i];
120	my $rec = $records->[$i];
121	# I've seen things you people wouldn't believe. Attack ships on fire
122	# off the shoulder of Orion. I watched C-beams glitter in the dark near
123	# the Tannhauser gate. MARC records where 999$c doesn't match the
124	# biblionumber column. All those moments will be lost in time... like
125	# tears in rain...
126	$rec->delete_fields($rec->field('999'));
127	$rec->append_fields(MARC::Field->new('999','','','c' => $bibnum, 'd' => $bibnum));
128	}
129	}
130
131	sub _convert_marc_to_json {
132	my $self = shift;
133	my $records = shift;
134	my $importer =
135	Catmandu::Importer::MARC->new( records => $records, id => '999c' );
136	my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
137	$importer = $fixer->fix($importer);
138	return $importer;
139	}
140
141	1;
142
143	__END__
144
145	=head1 AUTHOR
146
147	=over 4
148
149	=item Chris Cormack C<< <chrisc@catalyst.net.nz> >>
150
151	=item Robin Sheat C<< <robin@catalyst.net.nz> >>
152
153	=back
154
155	=cut

Line 0 Link Here

(-)a/Koha/ElasticSearch/Search.pm (+230 lines)
		1	package Koha::ElasticSearch::Search;
		2
		3	# Copyright 2014 Catalyst IT
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	=head1 NAME
		21
		22	Koha::ElasticSearch::Search - search functions for Elasticsearch
		23
		24	=head1 SYNOPSIS
		25
		26	my $searcher = Koha::ElasticSearch::Search->new();
		27	my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
		28	my $query = $builder->build_query('perl');
		29	my $results = $searcher->search($query);
		30	print "There were " . $results->total . " results.\n";
		31	$results->each(sub {
		32	push @hits, @_[0];
		33	});
		34
		35	=head1 METHODS
		36
		37	=cut
		38
		39	use base qw(Koha::ElasticSearch);
		40	use Koha::ItemTypes;
		41
		42	use Catmandu::Store::ElasticSearch;
		43
		44	use Data::Dumper; #TODO remove
		45	use Carp qw(cluck);
		46
		47	Koha::ElasticSearch::Search->mk_accessors(qw( store ));
		48
		49	=head2 search
		50
		51	my $results = $searcher->search($query, $page, $count);
		52
		53	Run a search using the query. It'll return C<$count> results, starting at page
		54	C<$page> (C<$page> counts from 1, anything less that, or C<undef> becomes 1.)
		55
		56	C<%options> is a hash containing extra options:
		57
		58	=over 4
		59
		60	=item offset
		61
		62	If provided, this overrides the C<$page> value, and specifies the record as
		63	an offset (i.e. the number of the record to start with), rather than a page.
		64
65	=back
66
67	=cut
68
69	sub search {
70	my ($self, $query, $page, $count, %options) = @_;
71
72	my $params = $self->get_elasticsearch_params();
73	my %paging;
74	$paging{limit} = $count \|\| 20;
75	# ES doesn't want pages, it wants a record to start from.
76	if (exists $options{offset}) {
77	$paging{start} = $options{offset};
78	} else {
79	$page = (!defined($page) \|\| ($page <= 0)) ? 1 : $page - 1;
80	$paging{start} = $page * $paging{limit};
81	}
82	$self->store(
83	Catmandu::Store::ElasticSearch->new(
84	%$params,
85	trace_calls => 0,
86	)
87	);
88	my $results = $self->store->bag->search( %$query, %paging );
89	return $results;
90	}
91
92	=head2 search_compat
93
94	my ( $error, $results, $facets ) = $search->search_compat(
95	$query, $simple_query, \@sort_by, \@servers,
96	$results_per_page, $offset, $expanded_facet, $branches,
97	$query_type, $scan
98	)
99
100	A search interface somewhat compatible with L<C4::Search->getRecords>. Anything
101	that is returned in the query created by build_query_compat will probably
102	get ignored here.
103
104	=cut
105
106	sub search_compat {
107	my (
108	$self, $query, $simple_query, $sort_by,
109	$servers, $results_per_page, $offset, $expanded_facet,
110	$branches, $query_type, $scan
111	) = @_;
112
113	my %options;
114	$options{offset} = $offset;
115	my $results = $self->search($query, undef, $results_per_page, %options);
116
117	# Convert each result into a MARC::Record
118	my (@records, $index);
119	$index = $offset; # opac-search expects results to be put in the
120	# right place in the array, according to $offset
121	$results->each(sub {
122	# The results come in an array for some reason
123	my $marc_json = @_[0]->{record};
124	my $marc = $self->json2marc($marc_json);
125	$records[$index++] = $marc;
126	});
127	# consumers of this expect a name-spaced result, we provide the default
128	# configuration.
129	my %result;
130	$result{biblioserver}{hits} = $results->total;
131	$result{biblioserver}{RECORDS} = \@records;
132	return (undef, \%result, $self->_convert_facets($results->{facets}));
133	}
134
135	=head2 json2marc
136
137	my $marc = $self->json2marc($marc_json);
138
139	Converts the form of marc (based on its JSON, but as a Perl structure) that
140	Catmandu stores into a MARC::Record object.
141
142	=cut
143
144	sub json2marc {
145	my ( $self, $marcjson ) = @_;
146
147	my $marc = MARC::Record->new();
148	$marc->encoding('UTF-8');
149
150	# fields are like:
151	# [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]
152	# conveniently, this is the form that MARC::Field->new() likes
153	foreach $field (@$marcjson) {
154	next if @$field < 5; # Shouldn't be possible, but...
155	if ( $field->[0] eq 'LDR' ) {
156	$marc->leader( $field->[4] );
157	}
158	else {
159	my $marc_field = MARC::Field->new(@$field);
160	$marc->append_fields($marc_field);
161	}
162	}
163	return $marc;
164	}
165
166	=head2 _convert_facets
167
168	my $koha_facets = _convert_facets($es_facets);
169
170	Converts elasticsearch facets types to the form that Koha expects.
171	It expects the ES facet name to match the Koha type, for example C<itype>,
172	C<au>, C<su-to>, etc.
173
174	=cut
175
176	sub _convert_facets {
177	my ( $self, $es ) = @_;
178
179	return undef if !$es;
180
181	# These should correspond to the ES field names, as opposed to the CCL
182	# things that zebra uses.
183	my %type_to_label = (
184	author => 'Authors',
185	location => 'Location',
186	itype => 'ItemTypes',
187	se => 'Series',
188	subject => 'Topics',
189	'su-geo' => 'Places',
190	);
191
192	# We also have some special cases, e.g. itypes that need to show the
193	# value rather than the code.
194	my $itypes = Koha::ItemTypes->new();
195	my %special = ( itype => sub { $itypes->get_description_for_code(@_) }, );
196	my @res;
197	while ( ( $type, $data ) = each %$es ) {
198	next if !exists( $type_to_label{$type} );
199	my $facet = {
200	type_id => $type . '_id',
201	expand => $type,
202	expandable => 1, # TODO figure how that's supposed to work
203	"type_label_$type_to_label{$type}" => 1,
204	type_link_value => $type,
205	};
206	foreach my $term ( @{ $data->{terms} } ) {
207	my $t = $term->{term};
208	my $c = $term->{count};
209	if ( exists( $special{$type} ) ) {
210	$label = $special{$type}->($t);
211	}
212	else {
213	$label = $t;
214	}
215	push @{ $facet->{facets} }, {
216	facet_count => $c,
217	facet_link_value => $t,
218	facet_title_value => $t . " ($c)",
219	facet_label_value => $label, # TODO either truncate this,
220	# or make the template do it like it should anyway
221	type_link_value => $type,
222	};
223	}
224	push @res, $facet if exists $facet->{facets};
225	}
226	return \@res;
227	}
228
229
230	1;

Line 0 Link Here

(-)a/Koha/ItemType.pm (+70 lines)
		1	package Koha::ItemType;
		2
		3	# This represents a single itemtype
		4
		5	# Copyright 2014 Catalyst IT
		6	#
		7	# This file is part of Koha.
		8	#
		9	# Koha is free software; you can redistribute it and/or modify it under the
		10	# terms of the GNU General Public License as published by the Free Software
		11	# Foundation; either version 3 of the License, or (at your option) any later
		12	# version.
		13	#
		14	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		15	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		16	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		17	#
		18	# You should have received a copy of the GNU General Public License along
		19	# with Koha; if not, write to the Free Software Foundation, Inc.,
		20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		21
		22	=head1 NAME
		23
		24	Koha::ItemType - represents a single itemtype
		25
		26	=head1 DESCRIPTION
		27
		28	This contains the data relating to a single itemtype
		29
		30	=head1 SYNOPSIS
		31
		32	use Koha::ItemTypes;
		33	my $types = Koha::ItemTypes->new();
		34	my $type = $types->get_itemtype('CODE');
		35	print $type->code, $type->description, $type->rentalcharge,
		36	$type->imageurl, $type->summary, $type->checkinmsg,
		37	$type->checkinmsgtype;
		38
		39	Creating an instance of C<Koha::ItemType> without using L<Koha::ItemTypes>
		40	can be done simply by passing a hashref containing the values to C<new()>.
		41	Note when doing this that a value for C<itemtype> will become a value for
		42	C<code>.
		43
		44	=head1 FUNCTIONS
		45
		46	In addition to the read-only accessors mentioned above, the following functions
		47	exist.
		48
		49	=cut
		50
		51	use Modern::Perl;
		52
		53	use base qw(Class::Accessor);
		54
		55	# TODO can we make these auto-generate from the input hash so it doesn't
		56	# have to be updated when the database is?
		57	__PACKAGE__->mk_ro_accessors(
		58	qw(code description rentalcharge imageurl
		59	summary checkinmsg checkinmsgtype)
		60	);
		61
		62	sub new {
		63	my $class = shift @_;
		64
65	my %data = ( %{ $_[0] }, code => $_[0]->{itemtype} );
66	my $self = $class->SUPER::new( \%data );
67	return $self;
68	}
69
70	1;

Line 0 Link Here

(-)a/Koha/ItemTypes.pm (+113 lines)
		1	package Koha::ItemTypes;
		2
		3	# This contains the item types that the system knows about.
		4
		5	# Copyright 2014 Catalyst IT
		6	#
		7	# This file is part of Koha.
		8	#
		9	# Koha is free software; you can redistribute it and/or modify it under the
		10	# terms of the GNU General Public License as published by the Free Software
		11	# Foundation; either version 3 of the License, or (at your option) any later
		12	# version.
		13	#
		14	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		15	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		16	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		17	#
		18	# You should have received a copy of the GNU General Public License along
		19	# with Koha; if not, write to the Free Software Foundation, Inc.,
		20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		21
		22	=head1 NAME
		23
		24	Koha::ItemTypes - handles the item types that Koha knows about
		25
		26	=head1 DESCRIPTION
		27
		28	This contains functions to access the item types.
		29
		30	Note that any changes that happen to the database while this object is live
		31	may not be reflected, so best don't hold onto it for a long time
		32
		33	=cut
		34
		35	use Koha::Database;
		36	use Koha::ItemType;
		37	use Modern::Perl;
		38
		39	use Data::Dumper; # TODO remove
		40	use base qw(Class::Accessor);
		41
		42	__PACKAGE__->mk_accessors(qw());
		43
		44	=head1 FUNCTIONS
		45
		46	=head2 new
		47
		48	my $itypes = Koha::ItemTypes->new();
		49
		50	Creates a new instance of the object.
		51
		52	=cut
		53
		54	# Handled by Class::Accessor
		55
		56	=head2 get_itemtype
		57
		58	my @itype = $itypes->get_itemtype('CODE1', 'CODE2');
		59
		60	This returns a L<Koha::ItemType> object for each of the provided codes. For
		61	any that don't exist, an C<undef> is returned.
		62
		63	=cut
		64
65	sub get_itemtype {
66	my ($self, @codes) = @_;
67
68	my $schema = Koha::Database->new()->schema();
69	my @res;
70
71	foreach my $c (@codes) {
72	if (exists $self->{cached}{$c}) {
73	push @res, $self->{cached}{$c};
74	next;
75	}
76	my $rs = $schema->resultset('Itemtype')->search( { itemtype => $c } );
77	my $r = $rs->next;
78	if (!$r) {
79	push @res, undef;
80	next;
81	}
82	my %data = $r->get_inflated_columns;
83	my $it = Koha::ItemType->new(\%data);
84	push @res, $it;
85	$self->{cached}{$c} = $it;
86	}
87	if (wantarray) {
88	return @res;
89	} else {
90	return @res ? $res[0] : undef;
91	}
92	}
93
94	=head2 get_description_for_code
95
96	my $desc = $itypes->get_description_for_code($code);
97
98	This returns the description for an itemtype code. As a special case, if
99	there is no itemtype for this code, it'll return what it was given.
100
101	It is mostly as a convenience function rather than using L<get_itemtype>.
102
103	=cut
104
105	sub get_description_for_code {
106	my ($self, $code) = @_;
107
108	my $itype = $self->get_itemtype($code);
109	return $code if !$itype;
110	return $itype->description;
111	}
112
113	1;

Line 0 Link Here

(-)a/Koha/Schema/Result/ElasticsearchMapping.pm (+105 lines)
		1	use utf8;
		2	package Koha::Schema::Result::ElasticsearchMapping;
		3
		4	# Created by DBIx::Class::Schema::Loader
		5	# DO NOT MODIFY THE FIRST PART OF THIS FILE
		6
		7	=head1 NAME
		8
		9	Koha::Schema::Result::ElasticsearchMapping
		10
		11	=cut
		12
		13	use strict;
		14	use warnings;
		15
		16	use base 'DBIx::Class::Core';
		17
		18	=head1 TABLE: C<elasticsearch_mapping>
		19
		20	=cut
		21
		22	__PACKAGE__->table("elasticsearch_mapping");
		23
		24	=head1 ACCESSORS
		25
		26	=head2 id
		27
		28	data_type: 'integer'
		29	is_auto_increment: 1
		30	is_nullable: 0
		31
		32	=head2 mapping
		33
		34	data_type: 'varchar'
		35	is_nullable: 1
		36	size: 255
		37
		38	=head2 type
		39
		40	data_type: 'varchar'
		41	is_nullable: 1
		42	size: 255
		43
		44	=head2 facet
		45
		46	data_type: 'tinyint'
		47	default_value: 0
		48	is_nullable: 1
		49
		50	=head2 marc21
		51
		52	data_type: 'varchar'
		53	is_nullable: 1
		54	size: 255
		55
		56	=head2 unimarc
		57
		58	data_type: 'varchar'
		59	is_nullable: 1
		60	size: 255
		61
		62	=head2 normarc
		63
		64	data_type: 'varchar'
65	is_nullable: 1
66	size: 255
67
68	=cut
69
70	__PACKAGE__->add_columns(
71	"id",
72	{ data_type => "integer", is_auto_increment => 1, is_nullable => 0 },
73	"mapping",
74	{ data_type => "varchar", is_nullable => 1, size => 255 },
75	"type",
76	{ data_type => "varchar", is_nullable => 1, size => 255 },
77	"facet",
78	{ data_type => "tinyint", default_value => 0, is_nullable => 1 },
79	"marc21",
80	{ data_type => "varchar", is_nullable => 1, size => 255 },
81	"unimarc",
82	{ data_type => "varchar", is_nullable => 1, size => 255 },
83	"normarc",
84	{ data_type => "varchar", is_nullable => 1, size => 255 },
85	);
86
87	=head1 PRIMARY KEY
88
89	=over 4
90
91	=item * L</id>
92
93	=back
94
95	=cut
96
97	__PACKAGE__->set_primary_key("id");
98
99
100	# Created by DBIx::Class::Schema::Loader v0.07040 @ 2014-06-06 16:20:16
101	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:uGRmWU0rshP6awyLMQYJeQ
102
103
104	# You can replace this text with custom code or comments, and it will be preserved on regeneration
105	1;

Line 0 Link Here

(-)a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm (+498 lines)
		1	package Koha::SearchEngine::Elasticsearch::QueryBuilder;
		2
		3	# This file is part of Koha.
		4	#
		5	# Copyright 2014 Catalyst IT Ltd.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	=head1 NAME
		21
		22	Koha::SearchEngine::Elasticsearch::QueryBuilder - constructs elasticsearch
		23	query objects from user-supplied queries
		24
		25	=head1 DESCRIPTION
		26
		27	This provides the functions that take a user-supplied search query, and
		28	provides something that can be given to elasticsearch to get answers.
		29
		30	=head1 SYNOPSIS
		31
		32	use Koha::SearchEngine::Elasticsearch;
		33	$builder = Koha::SearchEngine::Elasticsearch->new();
		34	my $simple_query = $builder->build_query("hello");
		35	# This is currently undocumented because the original code is undocumented
		36	my $adv_query = $builder->build_advanced_query($indexes, $operands, $operators);
		37
		38	=head1 METHODS
		39
		40	=cut
		41
		42	use base qw(Class::Accessor);
		43	use List::MoreUtils qw/ each_array /;
		44	use Modern::Perl;
		45	use URI::Escape;
		46
		47	use Data::Dumper; # TODO remove
		48
		49	=head2 build_query
		50
		51	my $simple_query = $builder->build_query("hello", %options)
		52
		53	This will build a query that can be issued to elasticsearch from the provided
		54	string input. This expects a lucene style search form (see
		55	L<http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-syntax>
		56	for details.)
		57
		58	It'll make an attempt to respect the various query options.
		59
		60	Additional options can be provided with the C<%options> hash.
		61
		62	=over 4
		63
		64	=item sort
65
66	This should be an arrayref of hashrefs, each containing a C<field> and an
67	C<direction> (optional, defaults to C<asc>.) The results will be sorted
68	according to these values. Valid values for C<direction> are 'asc' and 'desc'.
69
70	=back
71
72	=cut
73
74	sub build_query {
75	my ( $self, $query, %options ) = @_;
76
77	my $stemming = C4::Context->preference("QueryStemming") \|\| 0;
78	my $auto_truncation = C4::Context->preference("QueryAutoTruncate") \|\| 0;
79	my $weight_fields = C4::Context->preference("QueryWeightFields") \|\| 0;
80	my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") \|\| 0;
81
82	$query = '*' unless defined $query;
83
84	my $res;
85	$res->{query} = {
86	query_string => {
87	query => $query,
88	fuzziness => $fuzzy_enabled ? 'auto' : '0',
89	default_operator => "AND",
90	default_field => "_all",
91	}
92	};
93
94	if ( $options{sort} ) {
95	foreach my $sort ( @{ $options{sort} } ) {
96	my ( $f, $d ) = @$sort{qw/ field direction /};
97	die "Invalid sort direction, $d"
98	if $d && ( $d ne 'asc' && $d ne 'desc' );
99	$d = 'asc' unless $d;
100
101	# TODO account for fields that don't have a 'phrase' type
102	push @{ $res->{sort} }, { "$f.phrase" => { order => $d } };
103	}
104	}
105
106	# See _convert_facets in Search.pm for how these get turned into
107	# things that Koha can use.
108	$res->{facets} = {
109	author => { terms => { field => "author__facet" } },
110	subject => { terms => { field => "subject__facet" } },
111	itype => { terms => { field => "itype__facet" } },
112	};
113	return $res;
114	}
115
116	=head2 build_browse_query
117
118	my $browse_query = $builder->build_browse_query($field, $query);
119
120	This performs a "starts with" style query on a particular field. The field
121	to be searched must have been indexed with an appropriate mapping as a
122	"phrase" subfield.
123
124	=cut
125
126	sub build_browse_query {
127	my ( $self, $field, $query ) = @_;
128
129	my $fuzzy_enabled = C4::Context->preference("QueryFuzzy") \|\| 0;
130
131	return { query => '*' } if !defined $query;
132
133	# TODO this should come from Koha::Elasticsearch
134	my %field_whitelist = (
135	title => 1,
136	author => 1,
137	);
138	$field = 'title' if !exists $field_whitelist{$field};
139
140	my $res = {
141	query => {
142	match_phrase_prefix => {
143	"$field.phrase" => {
144	query => $query,
145	operator => 'or',
146	fuzziness => $fuzzy_enabled ? 'auto' : '0',
147	}
148	}
149	},
150	sort => [ { "$field.phrase" => { order => "asc" } } ],
151	};
152	}
153
154	=head2 build_query_compat
155
156	my (
157	$error, $query, $simple_query, $query_cgi,
158	$query_desc, $limit, $limit_cgi, $limit_desc,
159	$stopwords_removed, $query_type
160	)
161	= $builder->build_query_compat( \@operators, \@operands, \@indexes,
162	\@limits, \@sort_by, $scan, $lang );
163
164	This handles a search using the same api as L<C4::Search::buildQuery> does.
165
166	A very simple query will go in with C<$operands> set to ['query'], and
167	C<$sort_by> set to ['pubdate_dsc']. This simple case will return with
168	C<$query> set to something that can perform the search, C<$simple_query>
169	set to just the search term, C<$query_cgi> set to something that can
170	reproduce this search, and C<$query_desc> set to something else.
171
172	=cut
173
174	sub build_query_compat {
175	my ( $self, $operators, $operands, $indexes, $orig_limits, $sort_by, $scan,
176	$lang )
177	= @_;
178
179	#die Dumper ( $self, $operators, $operands, $indexes, $limits, $sort_by, $scan, $lang );
180	my @sort_params = $self->_convert_sort_fields(@$sort_by);
181	my @index_params = $self->_convert_index_fields(@$indexes);
182	my $limits = $self->_fix_limit_special_cases($orig_limits);
183
184	# Merge the indexes in with the search terms and the operands so that
185	# each search thing is a handy unit.
186	unshift @$operators, undef; # The first one can't have an op
187	my @search_params;
188	my $ea = each_array( @$operands, @$operators, @index_params );
189	while ( my ( $oand, $otor, $index ) = $ea->() ) {
190	next if ( !defined($oand) \|\| $oand eq '' );
191	push @search_params, {
192	operand => $self->_clean_search_term($oand), # the search terms
193	operator => defined($otor) ? uc $otor : undef, # AND and so on
194	$index ? %$index : (),
195	};
196	}
197
198	# We build a string query from limits and the queries. An alternative
199	# would be to pass them separately into build_query and let it build
200	# them into a structured ES query itself. Maybe later, though that'd be
201	# more robust.
202	my $query_str = join( ' AND ',
203	join( ' ', $self->_create_query_string(@search_params) ),
204	$self->_join_queries( $self->_convert_index_strings(@$limits) ) );
205
206	# If there's no query on the left, let's remove the junk left behind
207	$query_str =~ s/^ AND //;
208	my %options;
209	$options{sort} = \@sort_params;
210	my $query = $self->build_query( $query_str, %options );
211
212	#die Dumper($query);
213	# We roughly emulate the CGI parameters of the zebra query builder
214	my $query_cgi = 'idx=kw&q=' . uri_escape( $operands->[0] ) if @$operands;
215	my $simple_query = $operands->[0] if @$operands == 1;
216	my $query_desc = $simple_query;
217	my $limit = 'and ' . join( ' and ', @$limits );
218	my $limit_cgi =
219	'&limit=' . join( '&limit=', map { uri_escape($_) } @$orig_limits );
220	my $limit_desc = "@$limits";
221
222	return (
223	undef, $query, $simple_query, $query_cgi, $query_desc,
224	$limit, $limit_cgi, $limit_desc, undef, undef
225	);
226	}
227
228	=head2 _convert_sort_fields
229
230	my @sort_params = _convert_sort_fields(@sort_by)
231
232	Converts the zebra-style sort index information into elasticsearch-style.
233
234	C<@sort_by> is the same as presented to L<build_query_compat>, and it returns
235	something that can be sent to L<build_query>.
236
237	=cut
238
239	sub _convert_sort_fields {
240	my ( $self, @sort_by ) = @_;
241
242	# Turn the sorting into something we care about.
243	my %sort_field_convert = (
244	acqdate => 'acqdate',
245	author => 'author',
246	call_number => 'callnum',
247	popularity => 'issues',
248	relevance => undef, # default
249	title => 'title',
250	pubdate => 'pubdate',
251	);
252	my %sort_order_convert =
253	( qw( dsc desc ), qw( asc asc ), qw( az asc ), qw( za desc ) );
254
255	# Convert the fields and orders, drop anything we don't know about.
256	grep { $_->{field} } map {
257	my ( $f, $d ) = split /_/;
258	{
259	field => $sort_field_convert{$f},
260	direction => $sort_order_convert{$d}
261	}
262	} @sort_by;
263	}
264
265	=head2 _convert_index_fields
266
267	my @index_params = $self->_convert_index_fields(@indexes);
268
269	Converts zebra-style search index notation into elasticsearch-style.
270
271	C<@indexes> is an array of index names, as presented to L<build_query_compat>,
272	and it returns something that can be sent to L<build_query>.
273
274	B<TODO>: this will pull from the elasticsearch mappings table to figure out
275	types.
276
277	=cut
278
279	our %index_field_convert = (
280	'kw' => '_all',
281	'ti' => 'title',
282	'au' => 'author',
283	'su' => 'subject',
284	'nb' => 'isbn',
285	'se' => 'title-series',
286	'callnum' => 'callnum',
287	'mc-itype' => 'itype',
288	'ln' => 'ln',
289	'branch' => 'homebranch',
290	'fic' => 'lf',
291	'mus' => 'rtype',
292	'aud' => 'ta',
293	);
294
295	sub _convert_index_fields {
296	my ( $self, @indexes ) = @_;
297
298	my %index_type_convert =
299	( __default => undef, phr => 'phrase', rtrn => 'right-truncate' );
300
301	# Convert according to our table, drop anything that doesn't convert
302	grep { $_->{field} } map {
303	my ( $f, $t ) = split /,/;
304	{
305	field => $index_field_convert{$f},
306	type => $index_type_convert{ $t // '__default' }
307	}
308	} @indexes;
309	}
310
311	=head2 _convert_index_strings
312
313	my @searches = $self->_convert_index_strings(@searches);
314
315	Similar to L<_convert_index_fields>, this takes strings of the form
316	B<field:search term> and rewrites the field from zebra-style to
317	elasticsearch-style. Anything it doesn't understand is returned verbatim.
318
319	=cut
320
321	sub _convert_index_strings {
322	my ( $self, @searches ) = @_;
323
324	my @res;
325	foreach my $s (@searches) {
326	next if $s eq '';
327	my ( $field, $term ) = $s =~ /^\s([\w,-]?):(.*)/;
328	unless ( defined($field) && defined($term) ) {
329	push @res, $s;
330	next;
331	}
332	my ($conv) = $self->_convert_index_fields($field);
333	unless ( defined($conv) ) {
334	push @res, $s;
335	next;
336	}
337	push @res, $conv->{field} . ":"
338	. $self->_modify_string_by_type( %$conv, operand => $term );
339	}
340	return @res;
341	}
342
343	=head2 _modify_string_by_type
344
345	my $str = $self->_modify_string_by_type(%index_field);
346
347	If you have a search term (operand) and a type (phrase, right-truncated), this
348	will convert the string to have the function in lucene search terms, e.g.
349	wrapping quotes around it.
350
351	=cut
352
353	sub _modify_string_by_type {
354	my ( $self, %idx ) = @_;
355
356	my $type = $idx{type} \|\| '';
357	my $str = $idx{operand};
358	return $str unless $str; # Empty or undef, we can't use it.
359
360	$str .= '*' if $type eq 'right-truncate';
361	$str = '"' . $str . '"' if $type eq 'phrase';
362	return $str;
363	}
364
365	=head2 _convert_index_strings_freeform
366
367	my $search = $self->_convert_index_strings_freeform($search);
368
369	This is similar to L<_convert_index_strings>, however it'll search out the
370	things to change within the string. So it can handle strings such as
371	C<(su:foo) AND (su:bar)>, converting the C<su> appropriately.
372
373	=cut
374
375	sub _convert_index_strings_freeform {
376	my ( $self, $search ) = @_;
377
378	while ( my ( $zeb, $es ) = each %index_field_convert ) {
379	$search =~ s/\b$zeb:/$es:/g;
380	}
381	return $search;
382	}
383
384	=head2 _join_queries
385
386	my $query_str = $self->_join_queries(@query_parts);
387
388	This takes a list of query parts, that might be search terms on their own, or
389	booleaned together, or specifying fields, or whatever, wraps them in
390	parentheses, and ANDs them all together. Suitable for feeding to the ES
391	query string query.
392
393	=cut
394
395	sub _join_queries {
396	my ( $self, @parts ) = @_;
397
398	@parts = grep { defined($_) && $_ ne '' } @parts;
399	return () unless @parts;
400	return $parts[0] if @parts < 2;
401	join ' AND ', map { "($_)" } @parts;
402	}
403
404	=head2 _make_phrases
405
406	my @phrased_queries = $self->_make_phrases(@query_parts);
407
408	This takes the supplied queries and forces them to be phrases by wrapping
409	quotes around them. It understands field prefixes, e.g. 'subject:' and puts
410	the quotes outside of them if they're there.
411
412	=cut
413
414	sub _make_phrases {
415	my ( $self, @parts ) = @_;
416	map { s/^\s(\w?:)(.*)$/$1"$2"/r } @parts;
417	}
418
419	=head2 _create_query_string
420
421	my @query_strings = $self->_create_query_string(@queries);
422
423	Given a list of hashrefs, it will turn them into a lucene-style query string.
424	The hash should contain field, type (both for the indexes), operator, and
425	operand.
426
427	=cut
428
429	sub _create_query_string {
430	my ( $self, @queries ) = @_;
431
432	map {
433	my $otor = $_->{operator} ? $_->{operator} . ' ' : '';
434	my $field = $_->{field} ? $_->{field} . ':' : '';
435
436	my $oand = $self->_modify_string_by_type(%$_);
437	"$otor($field$oand)";
438	} @queries;
439	}
440
441	=head2 _clean_search_term
442
443	my $term = $self->_clean_search_term($term);
444
445	This cleans a search term by removing any funny characters that may upset
446	ES and give us an error. It also calls L<_convert_index_strings_freeform>
447	to ensure those parts are correct.
448
449	=cut
450
451	sub _clean_search_term {
452	my ( $self, $term ) = @_;
453
454	$term = $self->_convert_index_strings_freeform($term);
455	$term =~ s/[{}]/"/g;
456	return $term;
457	}
458
459	=head2 _fix_limit_special_cases
460
461	my $limits = $self->_fix_limit_special_cases($limits);
462
463	This converts any special cases that the limit specifications have into things
464	that are more readily processable by the rest of the code.
465
466	The argument should be an arrayref, and it'll return an arrayref.
467
468	=cut
469
470	sub _fix_limit_special_cases {
471	my ( $self, $limits ) = @_;
472
473	my @new_lim;
474	foreach my $l (@$limits) {
475
476	# This is set up by opac-search.pl
477	if ( $l =~ /^yr,st-numeric,ge=/ ) {
478	my ( $start, $end ) =
479	( $l =~ /^yr,st-numeric,ge=(.) and yr,st-numeric,le=(.)$/ );
480	next unless defined($start) && defined($end);
481	push @new_lim, "copydate:[$start TO $end]";
482	}
483	elsif ( $l =~ /^yr,st-numeric=/ ) {
484	my ($date) = ( $l =~ /^yr,st-numeric=(.*)$/ );
485	next unless defined($date);
486	push @new_lim, "copydate:$date";
487	}
488	elsif ( $l =~ /^available$/ ) {
489	push @new_lim, 'onloan:false';
490	}
491	else {
492	push @new_lim, $l;
493	}
494	}
495	return \@new_lim;
496	}
497
498	1;

Lines 14-20 Link Here

(-)a/koha-tmpl/opac-tmpl/prog/en/modules/search/results.tt (-4 / +22 lines)
14		14
15	<script type="text/javascript">	15	<script type="text/javascript">
16	$(document).ready(function() {	16	$(document).ready(function() {
17	$('#bookbag_form').find("input").hide();	17	// $('#bookbag_form').find("input").hide();
18	$('#sort_by').change(function() {	18	$('#sort_by').change(function() {
19	$('#bookbag_form').submit();	19	$('#bookbag_form').submit();
20	});	20	});
Lines 47-52 Link Here
47	</div>	47	</div>
48	<div class="searchresults">	48	<div class="searchresults">
49	<form action="/cgi-bin/koha/opac-search.pl" method="get" name="bookbag_form" id="bookbag_form">	49	<form action="/cgi-bin/koha/opac-search.pl" method="get" name="bookbag_form" id="bookbag_form">
		50	[%# IF (browse) %]
		51	<label for="browse_field">Browse: </label>
		52	<select name="browse_field" id="browse_field">
		53	<option value="title">Title</option>
		54	<option value="author">Author</option>
		55	<option value="callnumber">Call Number</option>
		56	<option value="subject">Subject</option>
		57	<option value="isbn">ISBN</option>
		58	<option value="issn">ISSN</option>
		59	</select>
		60	<input type="hidden" name="type" value="browse" />
		61	<br />
		62	<label for="search_field">Query:</label>
		63	<input type="text" name="q" style="display:initial;" />
		64	<input type="submit" value="Browse" style="display:initial;" />
		65	[%# END %]
50	<!-- TABLE RESULTS START -->	66	<!-- TABLE RESULTS START -->
51	<table>	67	<table>
52	<thead>	68	<thead>
Lines 76-89 Link Here
76	</thead>	92	</thead>
77	<!-- Actual Search Results -->	93	<!-- Actual Search Results -->
78	<tbody>	94	<tbody>
		95	[% USE Dumper %]
79	[% FOREACH SEARCH_RESULT IN SEARCH_RESULTS %]	96	[% FOREACH SEARCH_RESULT IN SEARCH_RESULTS %]
		97	[% result =SEARCH_RESULT.item('_source') %]
80	<tr>	98	<tr>
81	<td>	99	<td>
82	<input type="checkbox" id="bib[% SEARCH_RESULT.biblionumber %]" name="biblionumber" value="[% SEARCH_RESULT.biblionumber %]" /> <label for="bib[% SEARCH_RESULT.biblionumber %]"></label>	100	<input type="checkbox" id="bib[% result.biblionumber %]" name="biblionumber" value="[% result.biblionumber %]" /> <label for="bib[% result.biblionumber %]"></label>
83	</td>	101	</td>
84	<td>	102	<td>
85	<a class="title" href="/cgi-bin/koha/opac-detail.pl?biblionumber=[% SEARCH_RESULT.biblionumber \|url %]" title="View details for this title">[% SEARCH_RESULT.title \|html %]</a>	103	<a class="title" href="/cgi-bin/koha/opac-detail.pl?biblionumber=[% result.biblionumber \|url %]" title="View details for this title">[% result.title \|html %]</a>
86	by <a href="/cgi-bin/koha/opac-search.pl?q=author:[% SEARCH_RESULT.author \|url %]" title="Search for works by this author" class="author">[% SEARCH_RESULT.author %]</a>	104	by <a href="/cgi-bin/koha/opac-search.pl?q=author:[% result.author \|url %]" title="Search for works by this author" class="author">[% result.author %]</a>
87	</td>	105	</td>
88	</tr>	106	</tr>
89	[% END %]	107	[% END %]

Line 0 Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (+148 lines)
		1	#!/usr/bin/perl
		2
		3	# This inserts records from a Koha database into elastic search
		4
		5	# Copyright 2014 Catalyst IT
		6	#
		7	# This file is part of Koha.
		8	#
		9	# Koha is free software; you can redistribute it and/or modify it under the
		10	# terms of the GNU General Public License as published by the Free Software
		11	# Foundation; either version 3 of the License, or (at your option) any later
		12	# version.
		13	#
		14	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		15	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		16	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		17	#
		18	# You should have received a copy of the GNU General Public License along
		19	# with Koha; if not, write to the Free Software Foundation, Inc.,
		20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		21
		22	=head1 NAME
		23
		24	rebuild_elastic_search.pl - inserts records from a Koha database into Elasticsearch
		25
		26	=head1 SYNOPSIS
		27
		28	B<rebuild_elastic_search.pl>
		29	[B<-c\|--commit>=C<count>]
		30	[B<-v\|--verbose>]
		31	[B<-h\|--help>]
		32	[B<--man>]
		33
		34	=head1 DESCRIPTION
		35
		36	=head1 OPTIONS
		37
		38	=over
		39
		40	=item B<-c\|--commit>=C<count>
		41
		42	Specify how many records will be batched up before they're added to Elasticsearch.
		43	Higher should be faster, but will cause more RAM usage. Default is 100.
		44
		45	=item B<-d\|--delete>
		46
		47	Delete the index and recreate it before indexing.
		48
		49	=item B<-b\|--biblionumber>
		50
		51	Only index the supplied biblionumber, mostly for testing purposes. May be
		52	repeated.
		53
		54	=item B<-v\|--verbose>
		55
		56	By default, this program only emits warnings and errors. This makes it talk
		57	more. Add more to make it even more wordy, in particular when debugging.
		58
		59	=item B<-h\|--help>
		60
		61	Help!
		62
		63	=item B<--man>
		64
65	Full documentation.
66
67	=cut
68
69	use autodie;
70	use Getopt::Long;
71	use Koha::Biblio;
72	use Koha::ElasticSearch::Indexer;
73	use MARC::Field;
74	use MARC::Record;
75	use Modern::Perl;
76	use Pod::Usage;
77
78	use Data::Dumper; # TODO remove
79
80	my $verbose = 0;
81	my $commit = 100;
82	my ($delete, $help, $man);
83	my (@biblionumbers);
84
85	GetOptions(
86	'c\|commit=i' => \$commit,
87	'd\|delete' => \$delete,
88	'b\|biblionumber=i' => \@biblionumbers,
89	'v\|verbose!' => \$verbose,
90	'h\|help' => \$help,
91	'man' => \$man,
92	);
93
94	pod2usage(1) if $help;
95	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
96
97	my $next;
98	if (@biblionumbers) {
99	$next = sub {
100	my $r = shift @biblionumbers;
101	return () unless defined $r;
102	return ($r, Koha::Biblio->get_marc_biblio($r, item_data => 1));
103	};
104	} else {
105	my $records = Koha::Biblio->get_all_biblios_iterator();
106	$next = sub {
107	$records->next();
108	}
109	}
110	my $indexer = Koha::ElasticSearch::Indexer->new({index => 'biblios' });
111	if ($delete) {
112	# We know it's safe to not recreate the indexer because update_index
113	# hasn't been called yet.
114	$indexer->delete_index();
115	}
116
117	my $count = 0;
118	my $commit_count = $commit;
119	my (@bibnums_buffer, @commit_buffer);
120	while (scalar(my ($bibnum, $rec) = $next->())) {
121	_log(1,"$bibnum\n");
122	$count++;
123
124	push @bibnums_buffer, $bibnum;
125	push @commit_buffer, $rec;
126	if (!(--$commit_count)) {
127	_log(2, "Committing...\n");
128	$indexer->update_index(\@bibnums_buffer, \@commit_buffer);
129	$commit_count = $commit;
130	@bibnums_buffer = ();
131	@commit_buffer = ();
132	}
133	}
134	# There are probably uncommitted records
135	$indexer->update_index(\@bibnums_buffer, \@commit_buffer);
136	_log(1, "$count records indexed.\n");
137
138	# Output progress information.
139	#
140	# _log($level, $msg);
141	#
142	# Will output $msg if the verbosity setting is set to $level or more. Will
143	# not include a trailing newline.
144	sub _log {
145	my ($level, $msg) = @_;
146
147	print $msg if ($verbose <= $level);
148	}

Line 0 Link Here

(-)a/opac/elasticsearch.pl (+102 lines)
		1	#!/usr/bin/perl
		2
		3	# Copyright 2013 Catalyst
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	use Modern::Perl;
		21
		22	use C4::Context;
		23	use CGI;
		24	use C4::Auth;
		25	use C4::Koha;
		26	use C4::Output;
		27
		28	# TODO this should use the moose thing that auto-picks.
		29	use Koha::SearchEngine::Elasticsearch::QueryBuilder;
		30	use Koha::ElasticSearch::Search;
		31
		32	my $cgi = new CGI;
		33
		34	my $template_name;
		35	my $template_type = "basic";
		36	if ( $cgi->param("idx") or $cgi->param("q") ) {
		37	$template_name = 'search/results.tt';
		38	}
		39	else {
		40	$template_name = 'search/advsearch.tt';
		41	$template_type = 'advsearch';
		42	}
		43
		44	# load the template
		45	my ( $template, $borrowernumber, $cookie ) = get_template_and_user(
		46	{
		47	template_name => $template_name,
		48	query => $cgi,
		49	type => "opac",
		50	authnotrequired => 1,
		51	}
		52	);
		53	my %template_params;
		54	my $format = $cgi->param("format") \|\| 'html';
		55
		56	# load the Type stuff
		57	my $itemtypes = GetItemTypes;
		58
		59	my $page = $cgi->param("page") \|\| 1;
		60	my $count =
		61	$cgi->param('count')
		62	\|\| C4::Context->preference('OPACnumSearchResults')
		63	\|\| 20;
		64	my $q = $cgi->param("q");
65
66	my $searcher = Koha::ElasticSearch::Search->new();
67	my $builder = Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
68	my $query;
69	if ($cgi->param('type') eq 'browse') {
70	$query = $builder->build_browse_query($cgi->param('browse_field') \|\| undef, $q );
71	$template_params{browse} = 1;
72	} else {
73	$query = $builder->build_query($q);
74	}
75	my $results = $searcher->search( $query, $page, $count );
76	#my $results = $searcher->search( { "match_phrase_prefix" => { "title" => "the" } } );
77
78	# This is temporary, but will do the job for now.
79	my @hits;
80	$results->each(sub {
81	push @hits, { _source => @_[0] };
82	});
83	# Make a list of the page numbers
84	my @pages = map { { page => $_, current => ($_ == ( $page \|\| 1)) } } 1 .. int($results->total / $count);
85	my $max_page = int($results->total / $count);
86	# Pager template params
87	$template->param(
88	SEARCH_RESULTS => \@hits,
89	PAGE_NUMBERS => \@pages,
90	total => $results->total,
91	previous_page => ( $page > 1 ? $page - 1 : undef ),
92	next_page => ( $page < $max_page ? $page + 1 : undef ),
93	follower_params => [
94	{ var => 'type', val => $cgi->param('type') },
95	{ var => 'q', val => $q },
96	{ var => 'count', val => $count },
97	],
98	%template_params,
99	);
100
101	my $content_type = ( $format eq 'rss' or $format eq 'atom' ) ? $format : 'html';
102	output_with_http_headers $cgi, $cookie, $template->output, $content_type;

Lines 28-41 use Modern::Perl; Link Here

(-)a/opac/opac-search.pl (-9 / +32 lines)
28	# to perform, etc.	28	# to perform, etc.
29	## load Koha modules	29	## load Koha modules
30	use C4::Context;	30	use C4::Context;
		31	use C4::Search;
31		32
32	my $searchengine = C4::Context->preference("SearchEngine");	33	use Data::Dumper; # TODO remove
33	if ( $searchengine =~ /^Solr$/ ) {
34	warn "We use Solr";
35	require 'opac/search.pl';
36	exit;
37	} elsif ( $searchengine =~ /^Zebra$/ ) {
38		34
		35	use Koha::SearchEngine::Elasticsearch::QueryBuilder;
		36	use Koha::ElasticSearch::Search;
		37	use Koha::SearchEngine::Zebra::QueryBuilder;
		38	use Koha::SearchEngine::Zebra::Search;
		39
		40	my $searchengine = C4::Context->preference("SearchEngine");
		41	my ($builder, $searcher);
		42	#$searchengine = 'Zebra'; # XXX
		43	for ( $searchengine ) {
		44	when ( /^Solr$/ ) {
		45	warn "We use Solr";
		46	require 'opac/search.pl';
		47	exit;
		48	}
		49	when ( /^Zebra$/ ) {
		50	$builder=Koha::SearchEngine::Zebra::QueryBuilder->new();
		51	$searcher=Koha::SearchEngine::Zebra::Search->new();
		52	}
		53	when (/^Elasticsearch$/) {
		54	# Should use the base QueryBuilder, but I don't have it wired up
		55	# for moose yet.
		56	$builder=Koha::SearchEngine::Elasticsearch::QueryBuilder->new();
		57	# $builder=Koha::SearchEngine::Zebra::QueryBuilder->new();
		58	$searcher=Koha::ElasticSearch::Search->new({index => 'biblios'});
		59	}
39	}	60	}
40		61
41	use C4::Output;	62	use C4::Output;
Lines 449-455 my ($error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_ Link Here
449	my @results;	470	my @results;
450		471
451	## I. BUILD THE QUERY	472	## I. BUILD THE QUERY
452	( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type) = buildQuery(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang);	473	( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type) = $builder->build_query_compat(\@operators,\@operands,\@indexes,\@limits,\@sort_by, 0, $lang);
		474	#die Dumper( $error,$query,$simple_query,$query_cgi,$query_desc,$limit,$limit_cgi,$limit_desc,$stopwords_removed,$query_type);
453		475
454	sub _input_cgi_parse {	476	sub _input_cgi_parse {
455	my @elements;	477	my @elements;
Lines 528-538 if ($tag) { Link Here
528	$pasarParams .= '&simple_query=' . $simple_query;	550	$pasarParams .= '&simple_query=' . $simple_query;
529	$pasarParams .= '&query_type=' . $query_type if ($query_type);	551	$pasarParams .= '&query_type=' . $query_type if ($query_type);
530	eval {	552	eval {
531	($error, $results_hashref, $facets) = getRecords($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan,1);	553	($error, $results_hashref, $facets) = $searcher->search_compat($query,$simple_query,\@sort_by,\@servers,$results_per_page,$offset,$expanded_facet,$branches,$itemtypes,$query_type,$scan,1);
532	};	554	};
533	}	555	}
		556
534	# This sorts the facets into alphabetical order	557	# This sorts the facets into alphabetical order
535	if ($facets) {	558	if ($facets && @$facets) {
536	foreach my $f (@$facets) {	559	foreach my $f (@$facets) {
537	$f->{facets} = [ sort { uc($a->{facet_title_value}) cmp uc($b->{facet_title_value}) } @{ $f->{facets} } ];	560	$f->{facets} = [ sort { uc($a->{facet_title_value}) cmp uc($b->{facet_title_value}) } @{ $f->{facets} } ];
538	}	561	}

Line 0 Link Here

(-)a/t/Koha/ItemType.pm (+46 lines)
	1	#!/usr/bin/perl
	2	#
	3	# Copyright 2014 Catalyst IT
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it under the
	8	# terms of the GNU General Public License as published by the Free Software
	9	# Foundation; either version 3 of the License, or (at your option) any later
	10	# version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
	13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
	14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
	15	#
	16	# You should have received a copy of the GNU General Public License along
	17	# with Koha; if not, write to the Free Software Foundation, Inc.,
	18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
	19
	20	use Modern::Perl;
	21
	22	use Test::More tests => 8;
	23
	24	BEGIN {
	25	use_ok('Koha::ItemType');
	26	}
	27
	28	my $data = {
	29	itemtype => 'CODE',
	30	description => 'description',
	31	rentalcharge => 'rentalcharge',
	32	imageurl => 'imageurl',
	33	summary => 'summary',
	34	checkinmsg => 'checkinmsg',
	35	checkinmsgtype => 'checkinmsgtype',
	36	};
	37
	38	my $type = Koha::ItemType->new($data);
	39
	40	is( $type->code, 'CODE', 'itemtype/code' );
	41	is( $type->description, 'description', 'description' );
	42	is( $type->rentalcharge, 'rentalcharge', 'rentalcharge' );
	43	is( $type->imageurl, 'imageurl', 'imageurl' );
	44	is( $type->summary, 'summary', 'summary' );
	45	is( $type->checkinmsg, 'checkinmsg', 'checkinmsg' );
	46	is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );

Line 0 Link Here

(-)a/t/Koha_ElasticSearch_Indexer.t (+51 lines)
	1	#
	2	#===============================================================================
	3	#
	4	# FILE: Koha_ElasticSearch_Indexer.t
	5	#
	6	# DESCRIPTION:
	7	#
	8	# FILES: ---
	9	# BUGS: ---
	10	# NOTES: ---
	11	# AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
	12	# ORGANIZATION: Koha Development Team
	13	# VERSION: 1.0
	14	# CREATED: 09/12/13 08:57:25
	15	# REVISION: ---
	16	#===============================================================================
	17
	18	use strict;
	19	use warnings;
	20
	21	use Test::More tests => 5; # last test to print
	22	use MARC::Record;
	23
	24	use_ok('Koha::ElasticSearch::Indexer');
	25
	26	my $indexer;
	27	ok(
	28	my $indexer = Koha::ElasticSearch::Indexer->new(
	29	{
	30	'nodes' => ['localhost:9200'],
	31	'index' => 'mydb'
	32	}
	33	),
	34	'Creating new indexer object'
	35	);
	36
	37	my $marc_record = MARC::Record->new();
	38	my $field = MARC::Field->new( '001', '1234567' );
	39	$marc_record->append_fields($field);
	40	$field = MARC::Field->new( '020', '', '', 'a' => '1234567890123' );
	41	$marc_record->append_fields($field);
	42	$field = MARC::Field->new( '245', '', '', 'a' => 'Title' );
	43	$marc_record->append_fields($field);
	44
	45	my $records = [$marc_record];
	46	ok( my $converted = $indexer->convert_marc_to_json($records),
	47	'Convert some records' );
	48
	49	is( $converted->count, 1, 'One converted record' );
	50
	51	ok( $indexer->update_index($records), 'Update Index' );

Line 0 Link Here

(-)a/t/Koha_ElasticSearch_Search.t (+38 lines)
	1	#
	2	#===============================================================================
	3	#
	4	# FILE: Koha_ElasticSearch_Search.t
	5	#
	6	# DESCRIPTION:
	7	#
	8	# FILES: ---
	9	# BUGS: ---
	10	# NOTES: ---
	11	# AUTHOR: Chris Cormack (rangi), chrisc@catalyst.net.nz
	12	# ORGANIZATION: Koha Development Team
	13	# VERSION: 1.0
	14	# CREATED: 09/12/13 09:43:29
	15	# REVISION: ---
	16	#===============================================================================
	17
	18	use strict;
	19	use warnings;
	20
	21	use Test::More tests => 5; # last test to print
	22
	23	use_ok('Koha::ElasticSearch::Search');
	24
	25	ok(
	26	my $searcher = Koha::ElasticSearch::Search->new(
	27	{ 'nodes' => ['localhost:9200'], 'index' => 'mydb' }
	28	),
	29	'Creating a Koha::ElasticSearch::Search object'
	30	);
	31
	32	is( $searcher->index, 'mydb', 'Testing basic accessor' );
	33
	34	ok( $searcher->connect, 'Connect to ElasticSearch server' );
	35	ok( my $results = $searcher->search( { record => 'easy' } ), 'Do a search ' );
	36
	37	ok( my $marcresults = $searcher->marc_search( { record => 'Fish' } ),
	38	'Do a marc search' );

Line 0 Link Here

(-)a/t/db_dependent/Koha/ItemTypes.pm (-1 / +65 lines)
0	-	1	#!/usr/bin/perl
		2	#
		3	# Copyright 2014 Catalyst IT
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it under the
		8	# terms of the GNU General Public License as published by the Free Software
		9	# Foundation; either version 3 of the License, or (at your option) any later
		10	# version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		13	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		14	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		15	#
		16	# You should have received a copy of the GNU General Public License along
		17	# with Koha; if not, write to the Free Software Foundation, Inc.,
		18	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		19
		20	# XXX This doesn't work because I need to figure out how to do transactions
		21	# in a test-case with DBIx::Class
		22
		23	use Modern::Perl;
		24
		25	use Test::More tests => 8;
		26	use Data::Dumper;
		27
		28	BEGIN {
		29	use_ok('Koha::ItemTypes');
		30	}
		31
		32	my $dbh = C4::Context->dbh;
		33
		34	# Start transaction
		35	$dbh->{AutoCommit} = 0;
		36	$dbh->{RaiseError} = 1;
		37
		38	my $prep = $dbh->prepare('INSERT INTO itemtypes (itemtype, description, rentalcharge, imageurl, summary, checkinmsg, checkinmsgtype) VALUES (?,?,?,?,?,?,?)');
		39	$prep->execute('type1', 'description', 'rentalcharge', 'imageurl', 'summary', 'checkinmsg', 'checkinmsgtype');
		40	$prep->execute('type2', 'description', 'rentalcharge', 'imageurl', 'summary', 'checkinmsg', 'checkinmsgtype');
		41
		42	my $itypes = Koha::ItemTypes->new();
		43
		44	my @types = $itypes->get_itemtype('type1', 'type2');
		45
		46	die Dumper(\@types);
		47	my $type = $types[0];
		48	ok(defined($type), 'first result');
		49	is( $type->code, 'type1', 'itemtype/code' );
		50	is( $type->description, 'description', 'description' );
		51	is( $type->rentalcharge, 'rentalcharge', 'rentalcharge' );
		52	is( $type->imageurl, 'imageurl', 'imageurl' );
		53	is( $type->summary, 'summary', 'summary' );
		54	is( $type->checkinmsg, 'checkinmsg', 'checkinmsg' );
		55	is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );
		56
		57	$type = $types[1];
		58	ok(defined($type), 'second result');
		59	is( $type->code, 'type2', 'itemtype/code' );
		60	is( $type->description, 'description', 'description' );
		61	is( $type->rentalcharge, 'rentalcharge', 'rentalcharge' );
		62	is( $type->imageurl, 'imageurl', 'imageurl' );
		63	is( $type->summary, 'summary', 'summary' );
		64	is( $type->checkinmsg, 'checkinmsg', 'checkinmsg' );
		65	is( $type->checkinmsgtype, 'checkinmsgtype', 'checkinmsgtype' );