From 500e703d7b8ed3490a5a88bb5f629d3ecaefafc3 Mon Sep 17 00:00:00 2001 From: Olli-Antti Kivilahti Date: Wed, 11 Jan 2017 17:48:02 +0200 Subject: [PATCH] Bug 17885 - Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings throws DBD::mysql Duplicate entry exceptions When executing Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings() against populated search engine mappings tables, one gets this exception: DBD::mysql::st execute failed: Duplicate entry '388-73' for key 'PRIMARY' [for Statement \"INSERT INTO `search_marc_to_field` ( `facet`, `search_field_id`, `search_marc_map_id`, `sort`, `suggestible`) VALUES ( ?, ?, ?, ?, ? )\" with ParamValues: 0='', 1='73', 2='388', 3=undef, 4=''] at /usr/share/perl5/DBIx/Class/Storage/DBI.pm line 1834.\nDBIx::Class::Storage::DBI::_dbh_execute(): Duplicate entry '388-73' for key 'PRIMARY' at /home/koha/Koha/Koha/SearchField.pm line 38" This patch fixes this, by upserting those entries instead of adding them. Also adds a Koha::SearchMappingManager to manage the complexities of CRUD:ing these search mappings Also adds nice tests to make sure the Manager and the reset_elasticsearch_mappings() keep working as expected. --- Koha/SearchEngine/Elasticsearch.pm | 39 +++--- Koha/SearchMappingManager.pm | 149 +++++++++++++++++++++++ t/db_dependent/Koha/SearchEngine/ElasticSearch.t | 127 +++++++++++++++++++ 3 files changed, 293 insertions(+), 22 deletions(-) create mode 100644 Koha/SearchMappingManager.pm create mode 100644 t/db_dependent/Koha/SearchEngine/ElasticSearch.t diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 03cc5c3..d065706 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -24,12 +24,15 @@ use C4::Context; use Koha::Database; use Koha::SearchFields; use Koha::SearchMarcMaps; +use Koha::SearchMappingManager; use Carp; use JSON; use Modern::Perl; use Readonly; use YAML::Syck; +use Try::Tiny; +use Scalar::Util qw(blessed); use Data::Dumper; # TODO remove @@ -278,6 +281,7 @@ sub _elasticsearch_mapping_for_default { sub reset_elasticsearch_mappings { my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; my $indexes = LoadFile( $mappings_yaml ); + my $marcflavour = C4::Context->preference('marcflavour') or die "reset_elasticsearch_mappings():> System preferene 'marcflavour not set'"; while ( my ( $index_name, $fields ) = each %$indexes ) { while ( my ( $field_name, $data ) = each %$fields ) { @@ -286,8 +290,19 @@ sub reset_elasticsearch_mappings { my $mappings = $data->{mappings}; my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); for my $mapping ( @$mappings ) { + next unless (lc($marcflavour) eq lc($mapping->{marc_type})); #Don't push all marc mappings for marc flavours we don't need + my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); - $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } ); + + ##update_or_create + my $p = { + search_field_id => $search_field->id, + search_marc_map_id => $marc_field->id, + }; + $p->{facet} = $mapping->{facet} if $mapping->{facet}; + $p->{suggestible} = $mapping->{suggestible} if $mapping->{suggestible}; + $p->{sort} = $mapping->{sort} if $mapping->{sort}; + Koha::Database->schema->resultset('SearchMarcToField')->update_or_create($p); } } } @@ -421,27 +436,7 @@ sub _foreach_mapping { my ( $self, $sub ) = @_; # TODO use a caching framework here - my $search_fields = Koha::Database->schema->resultset('SearchField')->search( - { - 'search_marc_map.index_name' => $self->index, - }, - { join => { search_marc_to_fields => 'search_marc_map' }, - '+select' => [ - 'search_marc_to_fields.facet', - 'search_marc_to_fields.suggestible', - 'search_marc_to_fields.sort', - 'search_marc_map.marc_type', - 'search_marc_map.marc_field', - ], - '+as' => [ - 'facet', - 'suggestible', - 'sort', - 'marc_type', - 'marc_field', - ], - } - ); + my $search_fields = Koha::SearchMappingManager::get_search_mappings(); while ( my $search_field = $search_fields->next ) { $sub->( diff --git a/Koha/SearchMappingManager.pm b/Koha/SearchMappingManager.pm new file mode 100644 index 0000000..eb7d0c2 --- /dev/null +++ b/Koha/SearchMappingManager.pm @@ -0,0 +1,149 @@ +package Koha::SearchMappingManager; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Carp; + +use Koha::Database; + +=head1 NAME + +Koha::SearchMappingManager - Manager for operating on search field mappings + +=head1 SYNOPSIS + +This class helps to interface with the complex internals of the koha.search_*-tables +and their respective Koha::Objects in a correct manner. + +=cut + + + + +=head2 get_search_mappings + + my $search_fields = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios|authorities'}); + + while ( my $search_field = $search_fields->next ) { + $sub->( + $search_field->get_column('name'), + $search_field->get_column('type'), + $search_field->get_column('facet'), + $search_field->get_column('suggestible'), + $search_field->get_column('sort'), + $search_field->get_column('marc_type'), + $search_field->get_column('marc_field'), + ); + } + +Get each entry from the searchengine mappings tables. + +@PARAMS HASHRef of keys: + index_name => 'biblios|authorities' #Which Koha record types to look for + name => 'title' #The koha.search_fields.name +@RETURNS Koha::Schma::Resultset::SearchField with enriched fields from other relevant tables +@THROWS die when parameters are not properly given. Should throw a Koha::Exception::BadParameter, + but pushing all those Exceptions to the community version would take ages. + +=cut + +sub get_search_mappings { + my ($params) = @_; + die "get_search_mappings():> parameter 'index_name' is missing" unless ($params->{index_name}); + + my $search = { + 'search_marc_map.index_name' => $params->{index_name}, + }; + $search->{'me.name'} = $params->{name} if $params->{name}; + + return Koha::Database->schema->resultset('SearchField')->search( + $search, + { join => { search_marc_to_fields => 'search_marc_map' }, + '+select' => [ + 'search_marc_to_fields.facet', + 'search_marc_to_fields.suggestible', + 'search_marc_to_fields.sort', + 'search_marc_map.marc_type', + 'search_marc_map.marc_field', + ], + '+as' => [ + 'facet', + 'suggestible', + 'sort', + 'marc_type', + 'marc_field', + ], + } + ); +} + +=head2 flush + + Koha::SearchMappingManager::flush(); + +Removes all entries from the Koha's search mapping tables + +=cut + +sub flush { + my $schema = Koha::Database->schema; + $schema->resultset('SearchField')->delete_all(); + $schema->resultset('SearchMarcMap')->delete_all(); + $schema->resultset('SearchMarcToField')->delete_all(); +} + + +=head2 add_mapping + + Koha::SearchMappingManager::add_mapping({name => 'ln-test', + label => 'original language', + type => 'keyword', + index_name => 'biblios', + marc_type => 'marc21', + marc_field => '024a', + facet => 1, + suggestible => 1, + sort => 1}); + +=cut + +sub add_mapping { + my ($params) = @_; + + my $search_field = Koha::SearchFields->find_or_create({ name => $params->{name} , + label => $params->{label}, + type => $params->{type} ,}, + { key => 'name' }); + + my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $params->{index_name}, + marc_type => $params->{marc_type} , + marc_field => $params->{marc_field}, }); + + ##update_or_create + my $p = { + search_field_id => $search_field->id, + search_marc_map_id => $marc_field->id, + }; + $p->{facet} = $params->{facet} if $params->{facet}; + $p->{suggestible} = $params->{suggestible} if $params->{suggestible}; + $p->{sort} = $params->{sort} if $params->{sort}; + Koha::Database->schema->resultset('SearchMarcToField')->update_or_create($p); +} + +1; + diff --git a/t/db_dependent/Koha/SearchEngine/ElasticSearch.t b/t/db_dependent/Koha/SearchEngine/ElasticSearch.t new file mode 100644 index 0000000..95bfbb8 --- /dev/null +++ b/t/db_dependent/Koha/SearchEngine/ElasticSearch.t @@ -0,0 +1,127 @@ +#!/usr/bin/perl + +# Copyright 2015 Catalyst IT +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl qw(2014); +use utf8; +use Test::More; + +use Koha::SearchEngine::Elasticsearch; +use Koha::SearchMappingManager; + + +subtest "Reset Elasticsearch mappings", \&reset_elasticsearch_mappings; +sub reset_elasticsearch_mappings { + my ($rv, $mappings, $count, $mapping); + + ok(1, 'Scenario: Reset Elasticsearch mappings to an empty database'); + #There might or might not be any mappings. Whatever the initial status is, make sure we start from empty tables + $rv = Koha::SearchMappingManager::flush(); + ok($rv, 'Given empty search mapping tables'); + eval { + $rv = Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings(); + }; + if ($@){ + ok(0, $@); + } + ok(not($@), 'When reset_elasticsearch_mappings() has been ran'); + + $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios'}); + $count = $mappings->count(); + ok($count, 'Then search mapping tables have been populated'); + + + + ok(1, 'Scenario: Reset Elasticsearch mappings when custom mappings already exist'); + $rv = Koha::SearchMappingManager::add_mapping({name => 'ln-test', + label => 'original language', + type => 'keyword', + index_name => 'biblios', + marc_type => 'marc21', + marc_field => '024a', + facet => 1, + suggestible => 1, + sort => 1}); + ok($rv, 'Given a mapping table with a custom search field'); + eval { + $rv = Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings(); + }; + if ($@){ + ok(0, $@); + } + ok(not($@), 'When reset_elasticsearch_mappings() has been ran'); + + $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios'}); + $count = $mappings->count(); + ok($count > 10, 'Then search mapping tables have been populated'); +} + +subtest "Get Elasticsearch mappings", \&get_search_mappings; +sub get_search_mappings { + my ($mappings, $mapping); + + ok(1, 'Scenario: Get a single search mapping by name'); + $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios', name => 'ff7-00'}); + ok($mappings, 'When a search mappings is fetched'); + $mapping = $mappings->next(); + is($mapping->get_column('name'), 'ff7-00', 'Then the search mapping "name" matches'); + is($mapping->get_column('type'), '', 'And the search mapping "type" matches'); + is($mapping->get_column('facet'), '0', 'And the search mapping "facet" matches'); + is($mapping->get_column('suggestible'), '0', 'And the search mapping "suggestible" matches'); + is($mapping->get_column('sort'), undef, 'And the search mapping "sort" matches'); + is($mapping->get_column('marc_type'), 'marc21', 'And the search mapping "marc_type" matches'); + is($mapping->get_column('marc_field'), '007_/1', 'And the search mapping "marc_field" matches'); + + ok(1, 'Scenario: Get all search mappings'); + $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios'}); + ok($mappings, 'When search mappings are fetched'); + ok($mappings->count() > 10, 'Then we have "'.$mappings->count().'" search mappings :)') +} + +subtest "Add a search mapping", \&add_mapping; +sub add_mapping { + my ($rv, $mappings, $mapping, $count); + + ok(1, "Scenario: Add the same mapping twice and hope for no duplicate mappings"); + $rv = Koha::SearchMappingManager::add_mapping({name => 'ln-test', + label => 'original language', + type => 'keyword', + index_name => 'biblios', + marc_type => 'marc21', + marc_field => '024a', + facet => 1, + suggestible => 1, + sort => 1}); + $rv = Koha::SearchMappingManager::add_mapping({name => 'ln-test', + label => 'original language', + type => 'keyword', + index_name => 'biblios', + marc_type => 'marc21', + marc_field => '024a', + facet => 1, + suggestible => 1, + sort => 1}); + ok(1, "When the same search mapping is added twice"); + + $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios', name => 'ln-test'}); + $count = $mappings->count(); + is($count, 1, "Then we received only one mapping from the database"); +} + +done_testing; + -- 2.7.4