Attachment #82256 for bug #19893

View | Details | Raw Unified | Return to bug 19893
Collapse All | Expand All

Lines 34-39 use Search::Elasticsearch; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-29 / +205 lines)
34	use Try::Tiny;	34	use Try::Tiny;
35	use YAML::Syck;	35	use YAML::Syck;
36		36
		37	use List::Util qw( sum0 reduce );
		38	use Search::Elasticsearch;
		39	use MARC::File::XML;
		40	use MIME::Base64;
		41	use Encode qw(encode);
		42
37	__PACKAGE__->mk_ro_accessors(qw( index ));	43	__PACKAGE__->mk_ro_accessors(qw( index ));
38	__PACKAGE__->mk_accessors(qw( sort_fields ));	44	__PACKAGE__->mk_accessors(qw( sort_fields ));
39		45
Lines 67-72 sub new { Link Here
67	return $self;	73	return $self;
68	}	74	}
69		75
		76	sub get_elasticsearch {
		77	my $self = shift @_;
		78	unless (defined $self->{elasticsearch}) {
		79	my $conf = $self->get_elasticsearch_params();
		80	$self->{elasticsearch} = Search::Elasticsearch->new(
		81	client => "5_0::Direct",
		82	nodes => $conf->{nodes},
		83	cxn_pool => 'Sniff'
		84	);
		85	}
		86	return $self->{elasticsearch};
		87	}
		88
70	=head2 get_elasticsearch_params	89	=head2 get_elasticsearch_params
71		90
72	my $params = $self->get_elasticsearch_params();	91	my $params = $self->get_elasticsearch_params();
Lines 281-328 sub sort_fields { Link Here
281	return $self->_sort_fields_accessor();	300	return $self->_sort_fields_accessor();
282	}	301	}
283		302
284	# Provides the rules for data conversion.	303	sub marc_records_to_documents {
285	sub get_fixer_rules {	304	my ($self, $records) = @_;
		305	my $rules = $self->get_marc_mapping_rules();
		306	my $control_fields_rules = $rules->{control_fields};
		307	my $data_fields_rules = $rules->{data_fields};
		308	my $marcflavour = lc C4::Context->preference('marcflavour');
		309	my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat');
		310
		311	my @record_documents;
		312
		313	sub _process_mappings {
		314	my ($mappings, $data, $record_document) = @_;
		315	foreach my $mapping (@{$mappings}) {
		316	my ($target, $options) = @{$mapping};
		317	# Copy (scalar) data since can have multiple targets
		318	# with differing options for (possibly) mutating data
		319	# so need a different copy for each
		320	my $_data = $data;
		321	$record_document->{$target} //= [];
		322	if (defined $options->{substr}) {
		323	my ($start, $length) = @{$options->{substr}};
		324	$_data = length($data) > $start ? substr $data, $start, $length : '';
		325	}
		326	if (defined $options->{value_callbacks}) {
		327	$_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}});
		328	}
		329	if (defined $options->{property}) {
		330	$_data = {
		331	$options->{property} => $_data
		332	}
		333	}
		334	push @{$record_document->{$target}}, $_data;
		335	}
		336	}
		337	foreach my $record (@{$records}) {
		338	my $record_document = {};
		339	my $mappings = $rules->{leader};
		340	if ($mappings) {
		341	_process_mappings($mappings, $record->leader(), $record_document);
		342	}
		343	foreach my $field ($record->fields()) {
		344	if($field->is_control_field()) {
		345	my $mappings = $control_fields_rules->{$field->tag()};
		346	if ($mappings) {
		347	_process_mappings($mappings, $field->data(), $record_document);
		348	}
		349	}
		350	else {
		351	my $subfields_mappings = $data_fields_rules->{$field->tag()};
		352	if ($subfields_mappings) {
		353	my $wildcard_mappings = $subfields_mappings->{'*'};
		354	foreach my $subfield ($field->subfields()) {
		355	my ($code, $data) = @{$subfield};
		356	my $mappings = $subfields_mappings->{$code} // [];
		357	if ($wildcard_mappings) {
		358	$mappings = [@{$mappings}, @{$wildcard_mappings}];
		359	}
		360	if (@{$mappings}) {
		361	_process_mappings($mappings, $data, $record_document);
		362	}
		363	}
		364	}
		365	}
		366	}
		367	foreach my $field (keys %{$rules->{defaults}}) {
		368	unless (defined $record_document->{$field}) {
369	$record_document->{$field} = $rules->{defaults}->{$field};
370	}
371	}
372	foreach my $field (@{$rules->{sum}}) {
373	if (defined $record_document->{$field}) {
374	# TODO: validate numeric? filter?
375	# TODO: Or should only accept fields without nested values?
376	# TODO: Quick and dirty, improve if needed
377	$record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
378	}
379	}
380	# TODO: Perhaps should check if $records_document non empty, but really should never be the case
381	$record->encoding('UTF-8');
382	if ($serialization_format eq 'base64ISO2709') {
383	$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc()));
384	}
385	else {
386	$record_document->{'marc_data'} = $record->as_xml_record($marcflavour);
387	}
388	my $id = $record->subfield('999', 'c');
389	push @record_documents, [$id, $record_document];
390	}
391	return \@record_documents;
392	}
393
394	# Provides the rules for marc to Elasticsearch JSON document conversion.
395	sub get_marc_mapping_rules {
286	my ($self) = @_;	396	my ($self) = @_;
287		397
288	my $marcflavour = lc C4::Context->preference('marcflavour');	398	my $marcflavour = lc C4::Context->preference('marcflavour');
289	my @rules;	399	my @rules;
290		400
291	$self->_foreach_mapping(	401	sub _field_mappings {
292	sub {	402	my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_;
293	my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;	403	my %mapping_defaults = ();
294	return if $marc_type ne $marcflavour;	404	my @mappings;
295	my $options ='';	405
		406	my $substr_args = undef;
		407	if ($range) {
		408	# TODO: use value_callback instead?
		409	my ($start, $end) = map(int, split /-/, $range, 2);
		410	$substr_args = [$start];
		411	push @{$substr_args}, (defined $end ? $end - $start + 1 : 1);
		412	}
		413	my $default_options = {};
		414	if ($substr_args) {
		415	$default_options->{substr} = $substr_args;
		416	}
		417
		418	# TODO: Should probably have per type value callback/hook
		419	# but hard code for now
		420	if ($target_type eq 'boolean') {
		421	$default_options->{value_callbacks} //= [];
		422	push @{$default_options->{value_callbacks}}, sub {
		423	my ($value) = @_;
		424	# Trim whitespace at both ends
		425	$value =~ s/^\s+\|\s+$//g;
		426	return $value ? 'true' : 'false';
		427	};
		428	}
296		429
297	push @rules, "marc_map('$marc_field','${name}.\$append', $options)";	430	my $mapping = [$target_name, $default_options];
298	if ($facet) {	431	push @mappings, $mapping;
299	push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)";	432
		433	my @suffixes = ();
		434	push @suffixes, 'facet' if $facet;
		435	push @suffixes, 'suggestion' if $suggestible;
		436	push @suffixes, 'sort' if !defined $sort \|\| $sort;
		437
		438	foreach my $suffix (@suffixes) {
		439	my $mapping = ["${target_name}__$suffix"];
		440	# Hack, fix later in less hideous manner
		441	if ($suffix eq 'suggestion') {
		442	push @{$mapping}, {%{$default_options}, property => 'input'};
300	}	443	}
301	if ($suggestible) {	444	else {
302	push @rules,	445	push @{$mapping}, $default_options;
303	#"marc_map('$marc_field','${name}__suggestion.input.\$append', '')"; #must not have nested data structures in .input
304	"marc_map('$marc_field','${name}__suggestion.input.\$append')";
305	}	446	}
306	if ( $type eq 'boolean' ) {	447	push @mappings, $mapping;
		448	}
		449	return @mappings;
		450	};
		451	my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/;
		452	my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/;
		453	my $rules = {
		454	'leader' => [],
		455	'control_fields' => {},
		456	'data_fields' => {},
		457	'sum' => [],
		458	'defaults' => {}
		459	};
		460
		461	$self->_foreach_mapping(sub {
		462	my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_;
		463	return if $marc_type ne $marcflavour;
		464
		465	if ($type eq 'sum') {
		466	push @{$rules->{sum}}, $name;
		467	}
		468	elsif($type eq 'boolean') {
		469	# boolean gets special handling, if value doesn't exist for a field,
		470	# it is set to false
		471	$rules->{defaults}->{$name} = 'false';
		472	}
307		473
308	# boolean gets special handling, basically if it doesn't exist,	474	if ($marc_field =~ $field_spec_regexp) {
309	# it's added and set to false. Otherwise we can't query it.	475	my $field_tag = $1;
310	push @rules,	476	my $subfields = defined $2 ? $2 : '*';
311	"unless exists('$name') add_field('$name', 0) end";	477	my $range = defined $3 ? $3 : undef;
312	}	478	if ($field_tag < 10) {
313	if ($type eq 'sum' ) {	479	$rules->{control_fields}->{$field_tag} //= [];
314	push @rules, "sum('$name')";	480	my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
		481	push @{$rules->{control_fields}->{$field_tag}}, @mappings;
315	}	482	}
316	if ($self->sort_fields()->{$name}) {	483	else {
317	if ($sort \|\| !defined $sort) {	484	$rules->{data_fields}->{$field_tag} //= {};
318	push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)";	485	foreach my $subfield (split //, $subfields) {
		486	$rules->{data_fields}->{$field_tag}->{$subfield} //= [];
		487	my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
		488	push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings;
319	}	489	}
320	}	490	}
321	}	491	}
322	);	492	elsif ($marc_field =~ $leader_regexp) {
323		493	my $range = defined $1 ? $1 : undef;
324	push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_');	494	my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range);
325	return \@rules;	495	push @{$rules->{leader}}, @mappings;
		496	}
		497	else {
		498	die("Invalid marc field: $marc_field");
		499	}
		500	});
		501	return $rules;
326	}	502	}
327		503
328	=head2 _foreach_mapping	504	=head2 _foreach_mapping




        $self->_sanitise_records($biblionums, $records);
    }

    $self->ensure_mappings_updated();
    $self->bulk_index($records);
    return 1;
}

sub bulk_index {
    my ($self, $records) = @_;
    my $conf = $self->get_elasticsearch_params();
    my $elasticsearch = $self->get_elasticsearch();
    my $documents = $self->marc_records_to_documents($records);
    my @body;

    foreach my $document_info (@{$documents}) {
        my ($id, $document) = @{$document_info};
        push @body, {
            index => {
                _id => $id
            }
        };
        push @body, $document;
    }
    if (@body) {
        my $response = $elasticsearch->bulk(
            index => $conf->{index_name},
            type => 'data', # is just hard coded in Indexer.pm?
            body => \@body
        );
    }
    # TODO: handle response
    #print Data::Dumper::Dumper( $from->to_array );
    $self->store->bag->add_many($from);
    $self->store->bag->commit;
    return 1;
}

sub ensure_mappings_updated {
    my ($self) = @_;
    unless ($self->{_mappings_updated}) {
        $self->update_mappings();
    }
}

sub update_mappings {
    my ($self) = @_;
    my $conf = $self->get_elasticsearch_params();
    my $elasticsearch = $self->get_elasticsearch();
    my $mappings = $self->get_elasticsearch_mappings();

    foreach my $type (keys %{$mappings}) {
        my $response = $elasticsearch->indices->put_mapping(
            index => $conf->{index_name},
            type => $type,
            body => {
                $type => $mappings->{$type}
            }
        );
        # TODO: process response, produce errors etc
    }
    $self->{_mappings_updated} = 1;
}

=head2 $indexer->update_index_background($biblionums, $records)

This has exactly the same API as C<update_index_background> however it'll

    $self->delete_index(@_);
}

=head2 $indexer->create_index();

Create an index on the Elasticsearch server.

=cut

sub create_index {
    my ($self) = @_;

    if (!$self->store) {
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
            )
        );
    }
    $self->store->bag->commit;
}

=head2 $indexer->drop_index();

Drops the index from the elasticsearch server. Calling C<update_index>


sub drop_index {
    my ($self) = @_;
    if ($self->index_exists) {
        my $conf = $self->get_elasticsearch_params();
        my $elasticsearch = $self->get_elasticsearch();
        my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
        # TODO: Handle response? Convert errors to exceptions/die
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
            )
        );
    }
}

sub create_index {
    my ($self) = @_;
    my $conf = $self->get_elasticsearch_params();
    my $settings = $self->get_elasticsearch_settings();
    my $elasticsearch = $self->get_elasticsearch();
    my $response = $elasticsearch->indices->create(
        index => $conf->{index_name},
        body => {
            settings => $settings
        }
    );
    # TODO: Handle response? Convert errors to exceptions/die
}

sub index_exists {
    my ($self) = @_;
    my $conf = $self->get_elasticsearch_params();
    my $elasticsearch = $self->get_elasticsearch();
    return $elasticsearch->indices->exists(
        index => $conf->{index_name},
    );
}

sub _sanitise_records {

    }
}

sub _convert_marc_to_json {
    my $self    = shift;
    my $records = shift;
    my $importer =
      Catmandu::Importer::MARC->new( records => $records, id => '999c' );
    my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
    $importer = $fixer->fix($importer);
    return $importer;
}

1;

__END__




use Koha::SearchEngine::Search;
use MARC::Record;
use Catmandu::Store::ElasticSearch;
use MARC::File::XML;
use Data::Dumper; #TODO remove
use Carp qw(cluck);
use MIME::Base64;

Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));


    my $results = $self->search($query, undef, $results_per_page, %options);

    # Convert each result into a MARC::Record
    my @records;
    # opac-search expects results to be put in the
    # right place in the array, according to $offset
    my $index = $offset;
    $results->each(sub {
        $records[$index++] = $self->decode_record_from_result(@_);
    });
            my $marc = $self->json2marc($marc_json);
            $records[$index++] = $marc;
        });
    # consumers of this expect a name-spaced result, we provide the default
    # configuration.
    my %result;

    $res->each(
        sub {
            my %result;
            my $record    = $_[0];
            my $marc_json = $record->{record};

            # I wonder if these should be real values defined in the mapping
            # rather than hard-coded conversions.
            my $record    = $_[0];
            # Handle legacy nested arrays indexed with splitting enabled.
            my $authid = $record->{ 'Local-number' }[0];
            $authid = @$authid[0] if (ref $authid eq 'ARRAY');

            $result{authid} = $authid;

            # TODO put all this info into the record at index time so we

            # it's not reproduced here yet.
            my $authtype           = $rs->single;
            my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
            my $marc               = $self->decode_record_from_result(@_);
            my $mainentry          = $marc->field($auth_tag_to_report);
            my $reported_tag;
            if ($mainentry) {

    my $results = $self->search($query, undef, $max_results, %options);
    my @records;
    $results->each(sub {
            my $marc = $self->decode_record_from_result(@_);
            my $marc_json = $_[0]->{record};
            my $marc = $self->json2marc($marc_json);
            push @records, $marc;
        });
    return (undef, \@records, $results->total);

    return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
}

=head2 decode_record_from_result
    my $marc_record = $self->decode_record_from_result(@result);
    my $marc = $self->json2marc($marc_json);

Extracts marc data from Elasticsearch result and decodes to MARC::Record object
Catmandu stores into a MARC::Record object.

=cut

sub decode_record_from_result {
    # Result is passed in as array, will get flattened
    # and first element will be $result
    my ( $self, $result ) = @_;
    if (C4::Context->preference('ElasticsearchMARCSerializationFormat') eq 'MARCXML') {
        return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
    }
    else {
        return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
    # [ '001', undef, undef, '_', 'a value' ]
    # conveniently, this is the form that MARC::Field->new() likes
    foreach my $field (@$marcjson) {
        next if @$field < 5;
        if ( $field->[0] eq 'LDR' ) {
            $marc->leader( $field->[4] );
        }
        else {
            my $tag = $field->[0];
            my $marc_field;
            if ( MARC::Field->is_controlfield_tag( $field->[0] ) ) {
                $marc_field = MARC::Field->new($field->[0], $field->[4]);
            } else {
                $marc_field = MARC::Field->new(@$field);
            }
            $marc->append_fields($marc_field);
        }
    }
    return $marc;
}

=head2 max_result_window

Lines 5-13 general: Link Here

(-)a/admin/searchengine/elasticsearch/field_config.yaml (-1 / +3 lines)
5	type: string	5	type: string
6	analyzer: analyser_standard	6	analyzer: analyser_standard
7	properties:	7	properties:
8	record:	8	marc_data:
9	store: true	9	store: true
10	type: text	10	type: text
		11	analyzer: keyword
		12	index: false
11	# Search fields	13	# Search fields
12	search:	14	search:
13	boolean:	15	boolean:

Lines 427-429 Administration: Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/admin.pref (+8 lines)
427	choices:	427	choices:
428	Zebra: Zebra	428	Zebra: Zebra
429	Elasticsearch: Elasticsearch	429	Elasticsearch: Elasticsearch
		430	-
		431	- "Use"
		432	- pref: ElasticsearchMARCSerializationFormat
		433	default: MARCXML
		434	choices:
		435	MARCXML: MARCXML
		436	base64ISO2709: base64ISO2709
		437	- "as serialization format for MARC records stored in Elasticsearch index. base64ISO2709 is faster and will use less space but have a maximum record length which could cause issues with very large records."

Lines 160-167 sub do_reindex { Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-1 / +6 lines)
160	my ( $next, $index_name ) = @_;	160	my ( $next, $index_name ) = @_;
161		161
162	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );	162	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
		163
163	if ($delete) {	164	if ($delete) {
164	$indexer->drop_index();	165	$indexer->drop_index() if $indexer->index_exists();
		166	$indexer->create_index();
		167	}
		168	elsif (!$indexer->index_exists()) {
		169	# Create index if does not exist
165	$indexer->create_index();	170	$indexer->create_index();
166	}	171	}
167		172

Lines 17-27 Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-1 / +239 lines)
17		17
18	use Modern::Perl;	18	use Modern::Perl;
19		19
20	use Test::More tests => 3;	20	use Test::More tests => 4;
21	use Test::Exception;	21	use Test::Exception;
22		22
23	use t::lib::Mocks;	23	use t::lib::Mocks;
24		24
		25	use Test::MockModule;
		26
		27	use MARC::Record;
		28
25	use Koha::SearchEngine::Elasticsearch;	29	use Koha::SearchEngine::Elasticsearch;
26		30
27	subtest '_read_configuration() tests' => sub {	31	subtest '_read_configuration() tests' => sub {
Lines 108-110 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here
108	$mappings = $es->get_elasticsearch_mappings();	112	$mappings = $es->get_elasticsearch_mappings();
109	is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );	113	is( $mappings->{data}{_all}{type}, 'string', 'Field mappings parsed correctly' );
110	};	114	};
		115
		116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
		117
		118	plan tests => 29;
		119
		120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
		121
		122	my @mappings = (
		123	{
		124	name => 'author',
		125	type => 'string',
		126	facet => 1,
		127	suggestible => 1,
		128	sort => undef,
		129	marc_type => 'marc21',
		130	marc_field => '100a',
		131	},
		132	{
		133	name => 'author',
		134	type => 'string',
		135	facet => 1,
		136	suggestible => 1,
		137	sort => 1,
		138	marc_type => 'marc21',
		139	marc_field => '110a',
		140	},
		141	{
		142	name => 'title',
		143	type => 'string',
		144	facet => 0,
		145	suggestible => 1,
		146	sort => 1,
		147	marc_type => 'marc21',
		148	marc_field => '245a',
		149	},
		150	{
		151	name => 'unimarc_title',
		152	type => 'string',
		153	facet => 0,
		154	suggestible => 1,
		155	sort => 1,
		156	marc_type => 'unimarc',
		157	marc_field => '245a',
		158	},
		159	{
		160	name => 'title',
		161	type => 'string',
		162	facet => 0,
		163	suggestible => undef,
		164	sort => 0,
		165	marc_type => 'marc21',
		166	marc_field => '220',
		167	},
		168	{
		169	name => 'sum_item_price',
		170	type => 'sum',
		171	facet => 0,
		172	suggestible => 0,
		173	sort => 0,
		174	marc_type => 'marc21',
		175	marc_field => '952g',
		176	},
		177	{
		178	name => 'items_withdrawn_status',
179	type => 'boolean',
180	facet => 0,
181	suggestible => 0,
182	sort => 0,
183	marc_type => 'marc21',
184	marc_field => '9520',
185	},
186	{
187	name => 'type_of_record',
188	type => 'string',
189	facet => 0,
190	suggestible => 0,
191	sort => 0,
192	marc_type => 'marc21',
193	marc_field => 'leader_/6',
194	},
195	{
196	name => 'type_of_record_and_bib_level',
197	type => 'string',
198	facet => 0,
199	suggestible => 0,
200	sort => 0,
201	marc_type => 'marc21',
202	marc_field => 'leader_/6-7',
203	},
204	);
205
206	my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
207	$se->mock('_foreach_mapping', sub {
208	my ($self, $sub) = @_;
209
210	foreach my $map (@mappings) {
211	$sub->(
212	$map->{name},
213	$map->{type},
214	$map->{facet},
215	$map->{suggestible},
216	$map->{sort},
217	$map->{marc_type},
218	$map->{marc_field}
219	);
220	}
221	});
222
223	my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
224
225	my $marc_record_1 = MARC::Record->new();
226	$marc_record_1->leader(' cam 22 a 4500');
227	$marc_record_1->append_fields(
228	MARC::Field->new('100', '', '', a => 'Author 1'),
229	MARC::Field->new('110', '', '', a => 'Corp Author'),
230	MARC::Field->new('210', '', '', a => 'Title 1'),
231	MARC::Field->new('245', '', '', a => 'Title: first record'),
232	MARC::Field->new('999', '', '', c => '1234567'),
233	# ' ' for testing trimming of white space in boolean value callback:
234	MARC::Field->new('952', '', '', 0 => ' ', g => '123.30'),
235	MARC::Field->new('952', '', '', 0 => 0, g => '127.20'),
236	);
237	my $marc_record_2 = MARC::Record->new();
238	$marc_record_2->leader(' cam 22 a 4500');
239	$marc_record_2->append_fields(
240	MARC::Field->new('100', '', '', a => 'Author 2'),
241	# MARC::Field->new('210', '', '', a => 'Title 2'),
242	# MARC::Field->new('245', '', '', a => 'Title: second record'),
243	MARC::Field->new('999', '', '', c => '1234568'),
244	MARC::Field->new('952', '', '', 0 => 1, g => 'string where should be numeric'),
245	);
246	my $records = [$marc_record_1, $marc_record_2];
247
248	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
249
250	my $docs = $see->marc_records_to_documents($records);
251
252	# First record:
253
254	is(scalar @{$docs}, 2, 'Two records converted to documents');
255
256	is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
257
258	is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
259	is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
260
261	is(scalar @{$docs->[0][1]->{author__sort}}, 2, 'First document author__sort field should have two values');
262	is_deeply($docs->[0][1]->{author__sort}, ['Author 1', 'Corp Author'], 'First document author__sort field should be set correctly');
263
264	is(scalar @{$docs->[0][1]->{title__sort}}, 1, 'First document title__sort field should have one value');
265	is_deeply($docs->[0][1]->{title__sort}, ['Title: first record'], 'First document title__sort field should be set correctly');
266
267	is(scalar @{$docs->[0][1]->{author__suggestion}}, 2, 'First document author__suggestion field should contain two values');
268	is_deeply(
269	$docs->[0][1]->{author__suggestion},
270	[
271	{
272	'input' => 'Author 1'
273	},
274	{
275	'input' => 'Corp Author'
276	}
277	],
278	'First document author__suggestion field should be set correctly'
279	);
280
281	is(scalar @{$docs->[0][1]->{title__suggestion}}, 1, 'First document title__suggestion field should contain one value');
282	is_deeply(
283	$docs->[0][1]->{title__suggestion},
284	[{ 'input' => 'Title: first record' }],
285	'First document title__suggestion field should be set correctly'
286	);
287
288	ok(!(defined $docs->[0][1]->{title__facet}), 'First document should have no title__facet field');
289
290	is(scalar @{$docs->[0][1]->{author__facet}}, 2, 'First document author__facet field should have two values');
291	is_deeply(
292	$docs->[0][1]->{author__facet},
293	['Author 1', 'Corp Author'],
294	'First document author__facet field should be set correctly'
295	);
296
297	is(scalar @{$docs->[0][1]->{items_withdrawn_status}}, 2, 'First document items_withdrawn_status field should have two values');
298	is_deeply(
299	$docs->[0][1]->{items_withdrawn_status},
300	['false', 'false'],
301	'First document items_withdrawn_status field should be set correctly'
302	);
303
304	is(
305	$docs->[0][1]->{sum_item_price},
306	'250.5',
307	'First document sum_item_price field should be set correctly'
308	);
309
310	ok(defined $docs->[0][1]->{marc_xml}, 'First document marc_xml field should be set');
311
312	is(scalar @{$docs->[0][1]->{type_of_record}}, 1, 'First document type_of_record field should have one value');
313	is_deeply(
314	$docs->[0][1]->{type_of_record},
315	['a'],
316	'First document type_of_record field should be set correctly'
317	);
318
319	is(scalar @{$docs->[0][1]->{type_of_record_and_bib_level}}, 1, 'First document type_of_record_and_bib_level field should have one value');
320	is_deeply(
321	$docs->[0][1]->{type_of_record_and_bib_level},
322	['am'],
323	'First document type_of_record_and_bib_level field should be set correctly'
324	);
325
326	# Second record:
327
328	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
329	is_deeply($docs->[1][1]->{author}, ['Author 2'], 'Second document author field should be set correctly');
330
331	is(scalar @{$docs->[1][1]->{items_withdrawn_status}}, 1, 'Second document items_withdrawn_status field should have one value');
332	is_deeply(
333	$docs->[1][1]->{items_withdrawn_status},
334	['true'],
335	'Second document items_withdrawn_status field should be set correctly'
336	);
337
338	is(
339	$docs->[1][1]->{sum_item_price},
340	0,
341	'Second document sum_item_price field should be set correctly'
342	);
343
344	# Mappings marc_type:
345
346	ok(!(defined $docs->[0][1]->{unimarc_title}), "No mapping when marc_type doesn't match marc flavour");
347
348	};




# Copyright 2015 Catalyst IT
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>

use Modern::Perl;

use Test::More tests => 2;
use Test::MockModule;

use t::lib::Mocks;
use MARC::Record;

my $schema = Koha::Database->schema;

use_ok('Koha::SearchEngine::Elasticsearch');

subtest 'get_fixer_rules() tests' => sub {

    plan tests => 49;

    $schema->storage->txn_begin;

    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );

    my @mappings;

    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
    $se->mock( '_foreach_mapping', sub {
        my ($self, $sub ) = @_;

        foreach my $map ( @mappings ) {
            $sub->(
                $map->{name},
                $map->{type},
                $map->{facet},
                $map->{suggestible},
                $map->{sort},
                $map->{marc_type},
                $map->{marc_field}
            );
        }
    });

    my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });

    @mappings = (
        {
            name => 'author',
            type => 'string',
            facet => 1,
            suggestible => 1,
            sort => undef,
            marc_type => 'marc21',
            marc_field => '100a',
        },
        {
            name => 'author',
            type => 'string',
            facet => 1,
            suggestible => 1,
            sort => 1,
            marc_type => 'marc21',
            marc_field => '110a',
        },
    );

    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
    my $result = $see->get_fixer_rules();
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
    is( $result->[3], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
    is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
    is( $result->[8], q{move_field(_id,es_id)});

    $mappings[0]->{type}  = 'boolean';
    $mappings[1]->{type}  = 'boolean';
    $result = $see->get_fixer_rules();
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
    is( $result->[3], q{unless exists('} . $mappings[0]->{name} . q{') add_field('} . $mappings[0]->{name} . q{', 0) end});
    is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
    is( $result->[8], q{unless exists('} . $mappings[1]->{name} . q{') add_field('} . $mappings[1]->{name} . q{', 0) end});
    is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
    is( $result->[10], q{move_field(_id,es_id)});

    $mappings[0]->{type}  = 'sum';
    $mappings[1]->{type}  = 'sum';
    $result = $see->get_fixer_rules();
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
    is( $result->[3], q{sum('} . $mappings[0]->{name} . q{')});
    is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
    is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
    is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
    is( $result->[8], q{sum('} . $mappings[1]->{name} . q{')});
    is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
    is( $result->[10], q{move_field(_id,es_id)});

    $mappings[0]->{type}  = 'string';
    $mappings[0]->{facet} = 0;
    $mappings[1]->{type}  = 'string';
    $mappings[1]->{facet} = 0;

    $result = $see->get_fixer_rules();
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
    is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
    is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
    is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
    is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
    is( $result->[6], q{move_field(_id,es_id)});

    $mappings[0]->{suggestible}  = 0;
    $mappings[1]->{suggestible}  = 0;

    $result = $see->get_fixer_rules();
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
    is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
    is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
    is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
    is( $result->[4], q{move_field(_id,es_id)});

    $mappings[0]->{sort}  = 0;
    $mappings[1]->{sort}  = undef;

    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
    $result = $see->get_fixer_rules();
    is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
    is( $result->[1], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
    is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
    is( $result->[3], q{move_field(_id,es_id)});

    t::lib::Mocks::mock_preference( 'marcflavour', 'UNIMARC' );

    $result = $see->get_fixer_rules();
    is( $result->[0], q{move_field(_id,es_id)});
    is( $result->[1], undef, q{No mapping when marc_type doesn't match marchflavour} );

    $schema->storage->txn_rollback;

};





use Modern::Perl;

use Test::More tests => 3;
use Test::MockModule;
use t::lib::Mocks;



my $marc_record = MARC::Record->new();
$marc_record->append_fields(
	MARC::Field->new( '001', '1234567' ),
	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
	MARC::Field->new( '245', '', '', 'a' => 'Title' )
);

my $records = [$marc_record];
ok( my $converted = $indexer->_convert_marc_to_json($records),
    'Convert some records' );

is( $converted->count, 1, 'One converted record' );

SKIP: {


    skip 'Elasticsearch configuration not available', 1
        if $@;

    ok( $indexer->update_index(undef, $records), 'Update Index' );
}

subtest 'create_index() tests' => sub {

    plan tests => 3;

    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
    $se->mock( 'get_elasticsearch_params', sub {
        my ($self, $sub ) = @_;

        my $method = $se->original( 'get_elasticsearch_params' );
        my $params = $method->( $self );
        $params->{index_name} .= '__test';
        return $params;
    });

    my $indexer;
    ok(
        $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
        'Creating a new indexer object'
    );
    ok(
        $indexer->create_index(),
        'Creating an index'
    );
    $indexer->drop_index();
    ok(
        $indexer->drop_index(),
        'Dropping the index'
    );
};

subtest '_convert_marc_to_json() tests' => sub {

    plan tests => 4;

    $schema->storage->txn_begin;

    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );

    my @mappings = (
        {
            name => 'author',
            type => 'string',
            facet => 1,
            suggestible => 1,
            sort => '~',
            marc_type => 'marc21',
            marc_field => '100a',
        },
        {
            name => 'author',
            type => 'string',
            facet => 1,
            suggestible => 1,
            sort => '~',
            marc_type => 'marc21',
            marc_field => '110a',
        },
    );


    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
    $se->mock( '_foreach_mapping', sub {
        my ($self, $sub ) = @_;

        foreach my $map ( @mappings ) {
            $sub->(
                $map->{name},
                $map->{type},
                $map->{facet},
                $map->{suggestible},
                $map->{sort},
                $map->{marc_type},
                $map->{marc_field}
            );
        }
    });

    my $marc_record = MARC::Record->new();
    $marc_record->append_fields(
        MARC::Field->new( '001', '1234567' ),
        MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
        MARC::Field->new( '100', '', '', 'a' => 'Author' ),
        MARC::Field->new( '110', '', '', 'a' => 'Corp Author' ),
        MARC::Field->new( '245', '', '', 'a' => 'Title' ),
    );
    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->append_fields(
        MARC::Field->new( '001', '1234567' ),
        MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
        MARC::Field->new( '100', '', '', 'a' => 'Author' ),
        MARC::Field->new( '245', '', '', 'a' => 'Title' ),
    );
    my @records = ( $marc_record, $marc_record_2 );

    my $importer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => 'biblios' })->_convert_marc_to_json( \@records );
    my $conv = $importer->next();
    is( $conv->{author}[0], "Author", "First mapped author should be 100a");
    is( $conv->{author}[1], "Corp Author", "Second mapped author should be 110a");

    $conv = $importer->next();
    is( $conv->{author}[0], "Author", "First mapped author should be 100a");
    is( scalar @{$conv->{author}} , 1, "We should map field only if exists, shouldn't add extra nulls");

    $schema->storage->txn_rollback;
};
- 

Return to bug 19893

Lines 66-89 sub update_index { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-60 / +81 lines)
66	$self->_sanitise_records($biblionums, $records);	66	$self->_sanitise_records($biblionums, $records);
67	}	67	}
68		68
69	my $from = $self->_convert_marc_to_json($records);	69	$self->ensure_mappings_updated();
70	if ( !$self->store ) {	70	$self->bulk_index($records);
71	my $params = $self->get_elasticsearch_params();	71	return 1;
72	$self->store(	72	}
73	Catmandu::Store::ElasticSearch->new(	73
74	%$params,	74	sub bulk_index {
75	index_settings => $self->get_elasticsearch_settings(),	75	my ($self, $records) = @_;
76	index_mappings => $self->get_elasticsearch_mappings(),	76	my $conf = $self->get_elasticsearch_params();
77	)	77	my $elasticsearch = $self->get_elasticsearch();
		78	my $documents = $self->marc_records_to_documents($records);
		79	my @body;
		80
		81	foreach my $document_info (@{$documents}) {
		82	my ($id, $document) = @{$document_info};
		83	push @body, {
		84	index => {
		85	_id => $id
		86	}
		87	};
		88	push @body, $document;
		89	}
		90	if (@body) {
		91	my $response = $elasticsearch->bulk(
		92	index => $conf->{index_name},
		93	type => 'data', # is just hard coded in Indexer.pm?
		94	body => \@body
78	);	95	);
79	}	96	}
80		97	# TODO: handle response
81	#print Data::Dumper::Dumper( $from->to_array );
82	$self->store->bag->add_many($from);
83	$self->store->bag->commit;
84	return 1;	98	return 1;
85	}	99	}
86		100
		101	sub ensure_mappings_updated {
		102	my ($self) = @_;
		103	unless ($self->{_mappings_updated}) {
		104	$self->update_mappings();
		105	}
		106	}
		107
		108	sub update_mappings {
		109	my ($self) = @_;
		110	my $conf = $self->get_elasticsearch_params();
		111	my $elasticsearch = $self->get_elasticsearch();
		112	my $mappings = $self->get_elasticsearch_mappings();
		113
		114	foreach my $type (keys %{$mappings}) {
		115	my $response = $elasticsearch->indices->put_mapping(
		116	index => $conf->{index_name},
		117	type => $type,
		118	body => {
		119	$type => $mappings->{$type}
		120	}
		121	);
		122	# TODO: process response, produce errors etc
		123	}
		124	$self->{_mappings_updated} = 1;
		125	}
		126
87	=head2 $indexer->update_index_background($biblionums, $records)	127	=head2 $indexer->update_index_background($biblionums, $records)
88		128
89	This has exactly the same API as C<update_index_background> however it'll	129	This has exactly the same API as C<update_index_background> however it'll
Lines 139-166 sub delete_index_background { Link Here
139	$self->delete_index(@_);	179	$self->delete_index(@_);
140	}	180	}
141		181
142	=head2 $indexer->create_index();
143
144	Create an index on the Elasticsearch server.
145
146	=cut
147
148	sub create_index {
149	my ($self) = @_;
150
151	if (!$self->store) {
152	my $params = $self->get_elasticsearch_params();
153	$self->store(
154	Catmandu::Store::ElasticSearch->new(
155	%$params,
156	index_settings => $self->get_elasticsearch_settings(),
157	index_mappings => $self->get_elasticsearch_mappings(),
158	)
159	);
160	}
161	$self->store->bag->commit;
162	}
163
164	=head2 $indexer->drop_index();	182	=head2 $indexer->drop_index();
165		183
166	Drops the index from the elasticsearch server. Calling C<update_index>	184	Drops the index from the elasticsearch server. Calling C<update_index>
Lines 170-191 after this will recreate it again. Link Here
170		188
171	sub drop_index {	189	sub drop_index {
172	my ($self) = @_;	190	my ($self) = @_;
173		191	if ($self->index_exists) {
174	if (!$self->store) {	192	my $conf = $self->get_elasticsearch_params();
175	# If this index doesn't exist, this will create it. Then it'll be	193	my $elasticsearch = $self->get_elasticsearch();
176	# deleted. That's not the end of the world however.	194	my $response = $elasticsearch->indices->delete(index => $conf->{index_name});
177	my $params = $self->get_elasticsearch_params();	195	# TODO: Handle response? Convert errors to exceptions/die
178	$self->store(
179	Catmandu::Store::ElasticSearch->new(
180	%$params,
181	index_settings => $self->get_elasticsearch_settings(),
182	index_mappings => $self->get_elasticsearch_mappings(),
183	)
184	);
185	}	196	}
186	my $store = $self->store;	197	}
187	$self->store(undef);	198
188	$store->drop();	199	sub create_index {
		200	my ($self) = @_;
		201	my $conf = $self->get_elasticsearch_params();
		202	my $settings = $self->get_elasticsearch_settings();
		203	my $elasticsearch = $self->get_elasticsearch();
		204	my $response = $elasticsearch->indices->create(
		205	index => $conf->{index_name},
		206	body => {
		207	settings => $settings
		208	}
		209	);
		210	# TODO: Handle response? Convert errors to exceptions/die
		211	}
		212
		213	sub index_exists {
		214	my ($self) = @_;
		215	my $conf = $self->get_elasticsearch_params();
		216	my $elasticsearch = $self->get_elasticsearch();
		217	return $elasticsearch->indices->exists(
		218	index => $conf->{index_name},
		219	);
189	}	220	}
190		221
191	sub _sanitise_records {	222	sub _sanitise_records {
Lines 209-224 sub _sanitise_records { Link Here
209	}	240	}
210	}	241	}
211		242
212	sub _convert_marc_to_json {
213	my $self = shift;
214	my $records = shift;
215	my $importer =
216	Catmandu::Importer::MARC->new( records => $records, id => '999c' );
217	my $fixer = Catmandu::Fix->new( fixes => $self->get_fixer_rules() );
218	$importer = $fixer->fix($importer);
219	return $importer;
220	}
221
222	1;	243	1;
223		244
224	__END__	245	__END__

Lines 49-57 use Koha::SearchEngine::QueryBuilder; Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Search.pm (-47 / +24 lines)
49	use Koha::SearchEngine::Search;	49	use Koha::SearchEngine::Search;
50	use MARC::Record;	50	use MARC::Record;
51	use Catmandu::Store::ElasticSearch;	51	use Catmandu::Store::ElasticSearch;
52		52	use MARC::File::XML;
53	use Data::Dumper; #TODO remove	53	use Data::Dumper; #TODO remove
54	use Carp qw(cluck);	54	use Carp qw(cluck);
		55	use MIME::Base64;
55		56
56	Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));	57	Koha::SearchEngine::Elasticsearch::Search->mk_accessors(qw( store ));
57		58
Lines 157-171 sub search_compat { Link Here
157	my $results = $self->search($query, undef, $results_per_page, %options);	158	my $results = $self->search($query, undef, $results_per_page, %options);
158		159
159	# Convert each result into a MARC::Record	160	# Convert each result into a MARC::Record
160	my (@records, $index);	161	my @records;
161	$index = $offset; # opac-search expects results to be put in the	162	# opac-search expects results to be put in the
162	# right place in the array, according to $offset	163	# right place in the array, according to $offset
		164	my $index = $offset;
163	$results->each(sub {	165	$results->each(sub {
164	# The results come in an array for some reason	166	$records[$index++] = $self->decode_record_from_result(@_);
165	my $marc_json = $_[0]->{record};	167	});
166	my $marc = $self->json2marc($marc_json);
167	$records[$index++] = $marc;
168	});
169	# consumers of this expect a name-spaced result, we provide the default	168	# consumers of this expect a name-spaced result, we provide the default
170	# configuration.	169	# configuration.
171	my %result;	170	my %result;
Lines 196-209 sub search_auth_compat { Link Here
196	$res->each(	195	$res->each(
197	sub {	196	sub {
198	my %result;	197	my %result;
199	my $record = $_[0];
200	my $marc_json = $record->{record};
201		198
202	# I wonder if these should be real values defined in the mapping	199	# I wonder if these should be real values defined in the mapping
203	# rather than hard-coded conversions.	200	# rather than hard-coded conversions.
		201	my $record = $_[0];
204	# Handle legacy nested arrays indexed with splitting enabled.	202	# Handle legacy nested arrays indexed with splitting enabled.
205	my $authid = $record->{ 'Local-number' }[0];	203	my $authid = $record->{ 'Local-number' }[0];
206	$authid = @$authid[0] if (ref $authid eq 'ARRAY');	204	$authid = @$authid[0] if (ref $authid eq 'ARRAY');
		205
207	$result{authid} = $authid;	206	$result{authid} = $authid;
208		207
209	# TODO put all this info into the record at index time so we	208	# TODO put all this info into the record at index time so we
Lines 219-225 sub search_auth_compat { Link Here
219	# it's not reproduced here yet.	218	# it's not reproduced here yet.
220	my $authtype = $rs->single;	219	my $authtype = $rs->single;
221	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";	220	my $auth_tag_to_report = $authtype ? $authtype->auth_tag_to_report : "";
222	my $marc = $self->json2marc($marc_json);	221	my $marc = $self->decode_record_from_result(@_);
223	my $mainentry = $marc->field($auth_tag_to_report);	222	my $mainentry = $marc->field($auth_tag_to_report);
224	my $reported_tag;	223	my $reported_tag;
225	if ($mainentry) {	224	if ($mainentry) {
Lines 338-346 sub simple_search_compat { Link Here
338	my $results = $self->search($query, undef, $max_results, %options);	337	my $results = $self->search($query, undef, $max_results, %options);
339	my @records;	338	my @records;
340	$results->each(sub {	339	$results->each(sub {
341	# The results come in an array for some reason	340	my $marc = $self->decode_record_from_result(@_);
342	my $marc_json = $_[0]->{record};
343	my $marc = $self->json2marc($marc_json);
344	push @records, $marc;	341	push @records, $marc;
345	});	342	});
346	return (undef, \@records, $results->total);	343	return (undef, \@records, $results->total);
Lines 361-403 sub extract_biblionumber { Link Here
361	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );	358	return Koha::SearchEngine::Search::extract_biblionumber( $searchresultrecord );
362	}	359	}
363		360
364	=head2 json2marc	361	=head2 decode_record_from_result
365		362	my $marc_record = $self->decode_record_from_result(@result);
366	my $marc = $self->json2marc($marc_json);
367		363
368	Converts the form of marc (based on its JSON, but as a Perl structure) that	364	Extracts marc data from Elasticsearch result and decodes to MARC::Record object
369	Catmandu stores into a MARC::Record object.
370		365
371	=cut	366	=cut
372		367
373	sub json2marc {	368	sub decode_record_from_result {
374	my ( $self, $marcjson ) = @_;	369	# Result is passed in as array, will get flattened
375		370	# and first element will be $result
376	my $marc = MARC::Record->new();	371	my ( $self, $result ) = @_;
377	$marc->encoding('UTF-8');	372	if (C4::Context->preference('ElasticsearchMARCSerializationFormat') eq 'MARCXML') {
378		373	return MARC::Record->new_from_xml($result->{marc_data}, 'UTF-8', uc C4::Context->preference('marcflavour'));
379	# fields are like:	374	}
380	# [ '245', '1', '2', 'a' => 'Title', 'b' => 'Subtitle' ]	375	else {
381	# or	376	return MARC::Record->new_from_usmarc(decode_base64($result->{marc_data}));
382	# [ '001', undef, undef, '_', 'a value' ]
383	# conveniently, this is the form that MARC::Field->new() likes
384	foreach my $field (@$marcjson) {
385	next if @$field < 5;
386	if ( $field->[0] eq 'LDR' ) {
387	$marc->leader( $field->[4] );
388	}
389	else {
390	my $tag = $field->[0];
391	my $marc_field;
392	if ( MARC::Field->is_controlfield_tag( $field->[0] ) ) {
393	$marc_field = MARC::Field->new($field->[0], $field->[4]);
394	} else {
395	$marc_field = MARC::Field->new(@$field);
396	}
397	$marc->append_fields($marc_field);
398	}
399	}	377	}
400	return $marc;
401	}	378	}
402		379
403	=head2 max_result_window	380	=head2 max_result_window

Lines 1-164 Link Here

(-)a/t/db_dependent/Koha_Elasticsearch.t (-164 lines)
1	# Copyright 2015 Catalyst IT
2	#
3	# This file is part of Koha.
4	#
5	# Koha is free software; you can redistribute it and/or modify it
6	# under the terms of the GNU General Public License as published by
7	# the Free Software Foundation; either version 3 of the License, or
8	# (at your option) any later version.
9	#
10	# Koha is distributed in the hope that it will be useful, but
11	# WITHOUT ANY WARRANTY; without even the implied warranty of
12	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13	# GNU General Public License for more details.
14	#
15	# You should have received a copy of the GNU General Public License
16	# along with Koha; if not, see <http://www.gnu.org/licenses>
17
18	use Modern::Perl;
19
20	use Test::More tests => 2;
21	use Test::MockModule;
22
23	use t::lib::Mocks;
24	use MARC::Record;
25
26	my $schema = Koha::Database->schema;
27
28	use_ok('Koha::SearchEngine::Elasticsearch');
29
30	subtest 'get_fixer_rules() tests' => sub {
31
32	plan tests => 49;
33
34	$schema->storage->txn_begin;
35
36	t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
37
38	my @mappings;
39
40	my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
41	$se->mock( '_foreach_mapping', sub {
42	my ($self, $sub ) = @_;
43
44	foreach my $map ( @mappings ) {
45	$sub->(
46	$map->{name},
47	$map->{type},
48	$map->{facet},
49	$map->{suggestible},
50	$map->{sort},
51	$map->{marc_type},
52	$map->{marc_field}
53	);
54	}
55	});
56
57	my $see = Koha::SearchEngine::Elasticsearch->new({ index => 'biblios' });
58
59	@mappings = (
60	{
61	name => 'author',
62	type => 'string',
63	facet => 1,
64	suggestible => 1,
65	sort => undef,
66	marc_type => 'marc21',
67	marc_field => '100a',
68	},
69	{
70	name => 'author',
71	type => 'string',
72	facet => 1,
73	suggestible => 1,
74	sort => 1,
75	marc_type => 'marc21',
76	marc_field => '110a',
77	},
78	);
79
80	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
81	my $result = $see->get_fixer_rules();
82	is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
83	is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
84	is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
85	is( $result->[3], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
86	is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
87	is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
88	is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
89	is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
90	is( $result->[8], q{move_field(_id,es_id)});
91
92	$mappings[0]->{type} = 'boolean';
93	$mappings[1]->{type} = 'boolean';
94	$result = $see->get_fixer_rules();
95	is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
96	is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
97	is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
98	is( $result->[3], q{unless exists('} . $mappings[0]->{name} . q{') add_field('} . $mappings[0]->{name} . q{', 0) end});
99	is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
100	is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
101	is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
102	is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
103	is( $result->[8], q{unless exists('} . $mappings[1]->{name} . q{') add_field('} . $mappings[1]->{name} . q{', 0) end});
104	is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
105	is( $result->[10], q{move_field(_id,es_id)});
106
107	$mappings[0]->{type} = 'sum';
108	$mappings[1]->{type} = 'sum';
109	$result = $see->get_fixer_rules();
110	is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
111	is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__facet.$append', )});
112	is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
113	is( $result->[3], q{sum('} . $mappings[0]->{name} . q{')});
114	is( $result->[4], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
115	is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
116	is( $result->[6], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__facet.$append', )});
117	is( $result->[7], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
118	is( $result->[8], q{sum('} . $mappings[1]->{name} . q{')});
119	is( $result->[9], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
120	is( $result->[10], q{move_field(_id,es_id)});
121
122	$mappings[0]->{type} = 'string';
123	$mappings[0]->{facet} = 0;
124	$mappings[1]->{type} = 'string';
125	$mappings[1]->{facet} = 0;
126
127	$result = $see->get_fixer_rules();
128	is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
129	is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__suggestion.input.$append')});
130	is( $result->[2], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
131	is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
132	is( $result->[4], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__suggestion.input.$append')});
133	is( $result->[5], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
134	is( $result->[6], q{move_field(_id,es_id)});
135
136	$mappings[0]->{suggestible} = 0;
137	$mappings[1]->{suggestible} = 0;
138
139	$result = $see->get_fixer_rules();
140	is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
141	is( $result->[1], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{__sort.$append', )});
142	is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
143	is( $result->[3], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
144	is( $result->[4], q{move_field(_id,es_id)});
145
146	$mappings[0]->{sort} = 0;
147	$mappings[1]->{sort} = undef;
148
149	$see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first
150	$result = $see->get_fixer_rules();
151	is( $result->[0], q{marc_map('} . $mappings[0]->{marc_field} . q{','} . $mappings[0]->{name} . q{.$append', )});
152	is( $result->[1], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{.$append', )});
153	is( $result->[2], q{marc_map('} . $mappings[1]->{marc_field} . q{','} . $mappings[1]->{name} . q{__sort.$append', )});
154	is( $result->[3], q{move_field(_id,es_id)});
155
156	t::lib::Mocks::mock_preference( 'marcflavour', 'UNIMARC' );
157
158	$result = $see->get_fixer_rules();
159	is( $result->[0], q{move_field(_id,es_id)});
160	is( $result->[1], undef, q{No mapping when marc_type doesn't match marchflavour} );
161
162	$schema->storage->txn_rollback;
163
164	};

Lines 17-23 Link Here

(-)a/t/db_dependent/Koha_Elasticsearch_Indexer.t (-117 / +5 lines)
17		17
18	use Modern::Perl;	18	use Modern::Perl;
19		19
20	use Test::More tests => 7;	20	use Test::More tests => 3;
21	use Test::MockModule;	21	use Test::MockModule;
22	use t::lib::Mocks;	22	use t::lib::Mocks;
23		23
Lines 37-52 ok( Link Here
37		37
38	my $marc_record = MARC::Record->new();	38	my $marc_record = MARC::Record->new();
39	$marc_record->append_fields(	39	$marc_record->append_fields(
40	MARC::Field->new( '001', '1234567' ),	40	MARC::Field->new( '001', '1234567' ),
41	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),	41	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
42	MARC::Field->new( '245', '', '', 'a' => 'Title' )	42	MARC::Field->new( '245', '', '', 'a' => 'Title' )
43	);	43	);
44
45	my $records = [$marc_record];	44	my $records = [$marc_record];
46	ok( my $converted = $indexer->_convert_marc_to_json($records),
47	'Convert some records' );
48
49	is( $converted->count, 1, 'One converted record' );
50		45
51	SKIP: {	46	SKIP: {
52		47
Lines 55-165 SKIP: { Link Here
55	skip 'Elasticsearch configuration not available', 1	50	skip 'Elasticsearch configuration not available', 1
56	if $@;	51	if $@;
57		52
58	ok( $indexer->update_index(undef,$records), 'Update Index' );	53	ok( $indexer->update_index(undef, $records), 'Update Index' );
59	}	54	}
60
61	subtest 'create_index() tests' => sub {
62
63	plan tests => 3;
64
65	my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
66	$se->mock( 'get_elasticsearch_params', sub {
67	my ($self, $sub ) = @_;
68
69	my $method = $se->original( 'get_elasticsearch_params' );
70	my $params = $method->( $self );
71	$params->{index_name} .= '__test';
72	return $params;
73	});
74
75	my $indexer;
76	ok(
77	$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
78	'Creating a new indexer object'
79	);
80	ok(
81	$indexer->create_index(),
82	'Creating an index'
83	);
84	$indexer->drop_index();
85	ok(
86	$indexer->drop_index(),
87	'Dropping the index'
88	);
89	};
90
91	subtest '_convert_marc_to_json() tests' => sub {
92
93	plan tests => 4;
94
95	$schema->storage->txn_begin;
96
97	t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
98
99	my @mappings = (
100	{
101	name => 'author',
102	type => 'string',
103	facet => 1,
104	suggestible => 1,
105	sort => '~',
106	marc_type => 'marc21',
107	marc_field => '100a',
108	},
109	{
110	name => 'author',
111	type => 'string',
112	facet => 1,
113	suggestible => 1,
114	sort => '~',
115	marc_type => 'marc21',
116	marc_field => '110a',
117	},
118	);
119
120
121	my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
122	$se->mock( '_foreach_mapping', sub {
123	my ($self, $sub ) = @_;
124
125	foreach my $map ( @mappings ) {
126	$sub->(
127	$map->{name},
128	$map->{type},
129	$map->{facet},
130	$map->{suggestible},
131	$map->{sort},
132	$map->{marc_type},
133	$map->{marc_field}
134	);
135	}
136	});
137
138	my $marc_record = MARC::Record->new();
139	$marc_record->append_fields(
140	MARC::Field->new( '001', '1234567' ),
141	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
142	MARC::Field->new( '100', '', '', 'a' => 'Author' ),
143	MARC::Field->new( '110', '', '', 'a' => 'Corp Author' ),
144	MARC::Field->new( '245', '', '', 'a' => 'Title' ),
145	);
146	my $marc_record_2 = MARC::Record->new();
147	$marc_record_2->append_fields(
148	MARC::Field->new( '001', '1234567' ),
149	MARC::Field->new( '020', '', '', 'a' => '1234567890123' ),
150	MARC::Field->new( '100', '', '', 'a' => 'Author' ),
151	MARC::Field->new( '245', '', '', 'a' => 'Title' ),
152	);
153	my @records = ( $marc_record, $marc_record_2 );
154
155	my $importer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => 'biblios' })->_convert_marc_to_json( \@records );
156	my $conv = $importer->next();
157	is( $conv->{author}[0], "Author", "First mapped author should be 100a");
158	is( $conv->{author}[1], "Corp Author", "Second mapped author should be 110a");
159
160	$conv = $importer->next();
161	is( $conv->{author}[0], "Author", "First mapped author should be 100a");
162	is( scalar @{$conv->{author}} , 1, "We should map field only if exists, shouldn't add extra nulls");
163
164	$schema->storage->txn_rollback;
165	};
166	-