Bugzilla – Attachment 58825 Details for
Bug 17885
Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings throws DBD::mysql Duplicate entry exceptions
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 17885 - Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings throws DBD::mysql Duplicate entry exceptions
Bug-17885---KohaSearchEngineElasticsearch-resetela.patch (text/plain), 16.49 KB, created by
Olli-Antti Kivilahti
on 2017-01-11 15:53:19 UTC
(
hide
)
Description:
Bug 17885 - Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings throws DBD::mysql Duplicate entry exceptions
Filename:
MIME Type:
Creator:
Olli-Antti Kivilahti
Created:
2017-01-11 15:53:19 UTC
Size:
16.49 KB
patch
obsolete
>From 500e703d7b8ed3490a5a88bb5f629d3ecaefafc3 Mon Sep 17 00:00:00 2001 >From: Olli-Antti Kivilahti <olli-antti.kivilahti@jns.fi> >Date: Wed, 11 Jan 2017 17:48:02 +0200 >Subject: [PATCH] Bug 17885 - > Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings throws > DBD::mysql Duplicate entry exceptions > >When executing Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings() >against populated search engine mappings tables, one gets this exception: > >DBD::mysql::st execute failed: Duplicate entry '388-73' for key 'PRIMARY' >[for Statement \"INSERT INTO `search_marc_to_field` ( `facet`, `search_field_id`, > `search_marc_map_id`, `sort`, `suggestible`) VALUES ( ?, ?, ?, ?, ? )\" with ParamValues: >0='', 1='73', 2='388', 3=undef, 4=''] at /usr/share/perl5/DBIx/Class/Storage/DBI.pm >line 1834.\nDBIx::Class::Storage::DBI::_dbh_execute(): Duplicate entry '388-73' for key >'PRIMARY' at /home/koha/Koha/Koha/SearchField.pm line 38" > >This patch fixes this, by upserting those entries instead of adding them. > >Also adds a Koha::SearchMappingManager to manage the complexities of CRUD:ing these search mappings > >Also adds nice tests to make sure the Manager and the reset_elasticsearch_mappings() keep working >as expected. >--- > Koha/SearchEngine/Elasticsearch.pm | 39 +++--- > Koha/SearchMappingManager.pm | 149 +++++++++++++++++++++++ > t/db_dependent/Koha/SearchEngine/ElasticSearch.t | 127 +++++++++++++++++++ > 3 files changed, 293 insertions(+), 22 deletions(-) > create mode 100644 Koha/SearchMappingManager.pm > create mode 100644 t/db_dependent/Koha/SearchEngine/ElasticSearch.t > >diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm >index 03cc5c3..d065706 100644 >--- a/Koha/SearchEngine/Elasticsearch.pm >+++ b/Koha/SearchEngine/Elasticsearch.pm >@@ -24,12 +24,15 @@ use C4::Context; > use Koha::Database; > use Koha::SearchFields; > use Koha::SearchMarcMaps; >+use Koha::SearchMappingManager; > > use Carp; > use JSON; > use Modern::Perl; > use Readonly; > use YAML::Syck; >+use Try::Tiny; >+use Scalar::Util qw(blessed); > > use Data::Dumper; # TODO remove > >@@ -278,6 +281,7 @@ sub _elasticsearch_mapping_for_default { > sub reset_elasticsearch_mappings { > my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; > my $indexes = LoadFile( $mappings_yaml ); >+ my $marcflavour = C4::Context->preference('marcflavour') or die "reset_elasticsearch_mappings():> System preferene 'marcflavour not set'"; > > while ( my ( $index_name, $fields ) = each %$indexes ) { > while ( my ( $field_name, $data ) = each %$fields ) { >@@ -286,8 +290,19 @@ sub reset_elasticsearch_mappings { > my $mappings = $data->{mappings}; > my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); > for my $mapping ( @$mappings ) { >+ next unless (lc($marcflavour) eq lc($mapping->{marc_type})); #Don't push all marc mappings for marc flavours we don't need >+ > my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); >- $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } ); >+ >+ ##update_or_create >+ my $p = { >+ search_field_id => $search_field->id, >+ search_marc_map_id => $marc_field->id, >+ }; >+ $p->{facet} = $mapping->{facet} if $mapping->{facet}; >+ $p->{suggestible} = $mapping->{suggestible} if $mapping->{suggestible}; >+ $p->{sort} = $mapping->{sort} if $mapping->{sort}; >+ Koha::Database->schema->resultset('SearchMarcToField')->update_or_create($p); > } > } > } >@@ -421,27 +436,7 @@ sub _foreach_mapping { > my ( $self, $sub ) = @_; > > # TODO use a caching framework here >- my $search_fields = Koha::Database->schema->resultset('SearchField')->search( >- { >- 'search_marc_map.index_name' => $self->index, >- }, >- { join => { search_marc_to_fields => 'search_marc_map' }, >- '+select' => [ >- 'search_marc_to_fields.facet', >- 'search_marc_to_fields.suggestible', >- 'search_marc_to_fields.sort', >- 'search_marc_map.marc_type', >- 'search_marc_map.marc_field', >- ], >- '+as' => [ >- 'facet', >- 'suggestible', >- 'sort', >- 'marc_type', >- 'marc_field', >- ], >- } >- ); >+ my $search_fields = Koha::SearchMappingManager::get_search_mappings(); > > while ( my $search_field = $search_fields->next ) { > $sub->( >diff --git a/Koha/SearchMappingManager.pm b/Koha/SearchMappingManager.pm >new file mode 100644 >index 0000000..eb7d0c2 >--- /dev/null >+++ b/Koha/SearchMappingManager.pm >@@ -0,0 +1,149 @@ >+package Koha::SearchMappingManager; >+ >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it under the >+# terms of the GNU General Public License as published by the Free Software >+# Foundation; either version 3 of the License, or (at your option) any later >+# version. >+# >+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY >+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR >+# A PARTICULAR PURPOSE. See the GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License along >+# with Koha; if not, write to the Free Software Foundation, Inc., >+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. >+ >+use Modern::Perl; >+ >+use Carp; >+ >+use Koha::Database; >+ >+=head1 NAME >+ >+Koha::SearchMappingManager - Manager for operating on search field mappings >+ >+=head1 SYNOPSIS >+ >+This class helps to interface with the complex internals of the koha.search_*-tables >+and their respective Koha::Objects in a correct manner. >+ >+=cut >+ >+ >+ >+ >+=head2 get_search_mappings >+ >+ my $search_fields = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios|authorities'}); >+ >+ while ( my $search_field = $search_fields->next ) { >+ $sub->( >+ $search_field->get_column('name'), >+ $search_field->get_column('type'), >+ $search_field->get_column('facet'), >+ $search_field->get_column('suggestible'), >+ $search_field->get_column('sort'), >+ $search_field->get_column('marc_type'), >+ $search_field->get_column('marc_field'), >+ ); >+ } >+ >+Get each entry from the searchengine mappings tables. >+ >+@PARAMS HASHRef of keys: >+ index_name => 'biblios|authorities' #Which Koha record types to look for >+ name => 'title' #The koha.search_fields.name >+@RETURNS Koha::Schma::Resultset::SearchField with enriched fields from other relevant tables >+@THROWS die when parameters are not properly given. Should throw a Koha::Exception::BadParameter, >+ but pushing all those Exceptions to the community version would take ages. >+ >+=cut >+ >+sub get_search_mappings { >+ my ($params) = @_; >+ die "get_search_mappings():> parameter 'index_name' is missing" unless ($params->{index_name}); >+ >+ my $search = { >+ 'search_marc_map.index_name' => $params->{index_name}, >+ }; >+ $search->{'me.name'} = $params->{name} if $params->{name}; >+ >+ return Koha::Database->schema->resultset('SearchField')->search( >+ $search, >+ { join => { search_marc_to_fields => 'search_marc_map' }, >+ '+select' => [ >+ 'search_marc_to_fields.facet', >+ 'search_marc_to_fields.suggestible', >+ 'search_marc_to_fields.sort', >+ 'search_marc_map.marc_type', >+ 'search_marc_map.marc_field', >+ ], >+ '+as' => [ >+ 'facet', >+ 'suggestible', >+ 'sort', >+ 'marc_type', >+ 'marc_field', >+ ], >+ } >+ ); >+} >+ >+=head2 flush >+ >+ Koha::SearchMappingManager::flush(); >+ >+Removes all entries from the Koha's search mapping tables >+ >+=cut >+ >+sub flush { >+ my $schema = Koha::Database->schema; >+ $schema->resultset('SearchField')->delete_all(); >+ $schema->resultset('SearchMarcMap')->delete_all(); >+ $schema->resultset('SearchMarcToField')->delete_all(); >+} >+ >+ >+=head2 add_mapping >+ >+ Koha::SearchMappingManager::add_mapping({name => 'ln-test', >+ label => 'original language', >+ type => 'keyword', >+ index_name => 'biblios', >+ marc_type => 'marc21', >+ marc_field => '024a', >+ facet => 1, >+ suggestible => 1, >+ sort => 1}); >+ >+=cut >+ >+sub add_mapping { >+ my ($params) = @_; >+ >+ my $search_field = Koha::SearchFields->find_or_create({ name => $params->{name} , >+ label => $params->{label}, >+ type => $params->{type} ,}, >+ { key => 'name' }); >+ >+ my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $params->{index_name}, >+ marc_type => $params->{marc_type} , >+ marc_field => $params->{marc_field}, }); >+ >+ ##update_or_create >+ my $p = { >+ search_field_id => $search_field->id, >+ search_marc_map_id => $marc_field->id, >+ }; >+ $p->{facet} = $params->{facet} if $params->{facet}; >+ $p->{suggestible} = $params->{suggestible} if $params->{suggestible}; >+ $p->{sort} = $params->{sort} if $params->{sort}; >+ Koha::Database->schema->resultset('SearchMarcToField')->update_or_create($p); >+} >+ >+1; >+ >diff --git a/t/db_dependent/Koha/SearchEngine/ElasticSearch.t b/t/db_dependent/Koha/SearchEngine/ElasticSearch.t >new file mode 100644 >index 0000000..95bfbb8 >--- /dev/null >+++ b/t/db_dependent/Koha/SearchEngine/ElasticSearch.t >@@ -0,0 +1,127 @@ >+#!/usr/bin/perl >+ >+# Copyright 2015 Catalyst IT >+# >+# This file is part of Koha. >+# >+# Koha is free software; you can redistribute it and/or modify it >+# under the terms of the GNU General Public License as published by >+# the Free Software Foundation; either version 3 of the License, or >+# (at your option) any later version. >+# >+# Koha is distributed in the hope that it will be useful, but >+# WITHOUT ANY WARRANTY; without even the implied warranty of >+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the >+# GNU General Public License for more details. >+# >+# You should have received a copy of the GNU General Public License >+# along with Koha; if not, see <http://www.gnu.org/licenses>. >+ >+use Modern::Perl qw(2014); >+use utf8; >+use Test::More; >+ >+use Koha::SearchEngine::Elasticsearch; >+use Koha::SearchMappingManager; >+ >+ >+subtest "Reset Elasticsearch mappings", \&reset_elasticsearch_mappings; >+sub reset_elasticsearch_mappings { >+ my ($rv, $mappings, $count, $mapping); >+ >+ ok(1, 'Scenario: Reset Elasticsearch mappings to an empty database'); >+ #There might or might not be any mappings. Whatever the initial status is, make sure we start from empty tables >+ $rv = Koha::SearchMappingManager::flush(); >+ ok($rv, 'Given empty search mapping tables'); >+ eval { >+ $rv = Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings(); >+ }; >+ if ($@){ >+ ok(0, $@); >+ } >+ ok(not($@), 'When reset_elasticsearch_mappings() has been ran'); >+ >+ $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios'}); >+ $count = $mappings->count(); >+ ok($count, 'Then search mapping tables have been populated'); >+ >+ >+ >+ ok(1, 'Scenario: Reset Elasticsearch mappings when custom mappings already exist'); >+ $rv = Koha::SearchMappingManager::add_mapping({name => 'ln-test', >+ label => 'original language', >+ type => 'keyword', >+ index_name => 'biblios', >+ marc_type => 'marc21', >+ marc_field => '024a', >+ facet => 1, >+ suggestible => 1, >+ sort => 1}); >+ ok($rv, 'Given a mapping table with a custom search field'); >+ eval { >+ $rv = Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings(); >+ }; >+ if ($@){ >+ ok(0, $@); >+ } >+ ok(not($@), 'When reset_elasticsearch_mappings() has been ran'); >+ >+ $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios'}); >+ $count = $mappings->count(); >+ ok($count > 10, 'Then search mapping tables have been populated'); >+} >+ >+subtest "Get Elasticsearch mappings", \&get_search_mappings; >+sub get_search_mappings { >+ my ($mappings, $mapping); >+ >+ ok(1, 'Scenario: Get a single search mapping by name'); >+ $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios', name => 'ff7-00'}); >+ ok($mappings, 'When a search mappings is fetched'); >+ $mapping = $mappings->next(); >+ is($mapping->get_column('name'), 'ff7-00', 'Then the search mapping "name" matches'); >+ is($mapping->get_column('type'), '', 'And the search mapping "type" matches'); >+ is($mapping->get_column('facet'), '0', 'And the search mapping "facet" matches'); >+ is($mapping->get_column('suggestible'), '0', 'And the search mapping "suggestible" matches'); >+ is($mapping->get_column('sort'), undef, 'And the search mapping "sort" matches'); >+ is($mapping->get_column('marc_type'), 'marc21', 'And the search mapping "marc_type" matches'); >+ is($mapping->get_column('marc_field'), '007_/1', 'And the search mapping "marc_field" matches'); >+ >+ ok(1, 'Scenario: Get all search mappings'); >+ $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios'}); >+ ok($mappings, 'When search mappings are fetched'); >+ ok($mappings->count() > 10, 'Then we have "'.$mappings->count().'" search mappings :)') >+} >+ >+subtest "Add a search mapping", \&add_mapping; >+sub add_mapping { >+ my ($rv, $mappings, $mapping, $count); >+ >+ ok(1, "Scenario: Add the same mapping twice and hope for no duplicate mappings"); >+ $rv = Koha::SearchMappingManager::add_mapping({name => 'ln-test', >+ label => 'original language', >+ type => 'keyword', >+ index_name => 'biblios', >+ marc_type => 'marc21', >+ marc_field => '024a', >+ facet => 1, >+ suggestible => 1, >+ sort => 1}); >+ $rv = Koha::SearchMappingManager::add_mapping({name => 'ln-test', >+ label => 'original language', >+ type => 'keyword', >+ index_name => 'biblios', >+ marc_type => 'marc21', >+ marc_field => '024a', >+ facet => 1, >+ suggestible => 1, >+ sort => 1}); >+ ok(1, "When the same search mapping is added twice"); >+ >+ $mappings = Koha::SearchMappingManager::get_search_mappings({index_name => 'biblios', name => 'ln-test'}); >+ $count = $mappings->count(); >+ is($count, 1, "Then we received only one mapping from the database"); >+} >+ >+done_testing; >+ >-- >2.7.4
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 17885
:
58825
|
58940
|
58941
|
69338
|
78327
|
91308
|
97198