From 1976be11ad238afd6db98f700379a2df87a977db Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Mon, 12 Oct 2015 16:46:16 +0100 Subject: [PATCH] Bug 12478: Move mapping attributes to the join table To make things consistent, the attributes of the mappings (sort, suggestible, facet) should be moved to the join tables (search_marc_to_field). Indeed the search_marc_map could represent a mapping for several search fields with different attributes. Before this change it was not possible. --- Koha/ElasticSearch.pm | 53 ++++++++------ Koha/Objects.pm | 18 +++++ Koha/SearchField.pm | 5 ++ .../data/mysql/atomicupdate/elastic_search.perl | 83 ++++++++++++++++++++++ installer/data/mysql/updatedatabase.pl | 33 --------- 5 files changed, 137 insertions(+), 55 deletions(-) create mode 100644 installer/data/mysql/atomicupdate/elastic_search.perl diff --git a/Koha/ElasticSearch.pm b/Koha/ElasticSearch.pm index e91d5b8..d56c344 100644 --- a/Koha/ElasticSearch.pm +++ b/Koha/ElasticSearch.pm @@ -374,29 +374,38 @@ sub _foreach_mapping { my ( $self, $sub ) = @_; # TODO use a caching framework here - my $database = Koha::Database->new(); - my $schema = $database->schema(); - my $rs = - $schema->resultset('SearchMarcMap') - ->search( { index_name => $self->index } ); - for my $row ( $rs->all ) { - my $marc_type = $row->marc_type; - my $marc_field = $row->marc_field; - my $facet = $row->facet; - my $suggestible = $row->suggestible; - my $search_field = $row->search_fields(); - my $sort = $row->sort(); - for my $sf ( $search_field->all ) { - $sub->( - $sf->name, - $sf->type, - $facet, - $suggestible, - $sort, - $marc_type, - $marc_field, - ); + my $search_fields = Koha::Database->schema->resultset('SearchField')->search( + { + 'search_marc_map.index_name' => $self->index, + }, + { join => { search_marc_to_fields => 'search_marc_map' }, + '+select' => [ + 'search_marc_to_fields.facet', + 'search_marc_to_fields.suggestible', + 'search_marc_to_fields.sort', + 'search_marc_map.marc_type', + 'search_marc_map.marc_field', + ], + '+as' => [ + 'facet', + 'suggestible', + 'sort', + 'marc_type', + 'marc_field', + ], } + ); + + while ( my $search_field = $search_fields->next ) { + $sub->( + $search_field->name, + $search_field->type, + $search_field->get_column('facet'), + $search_field->get_column('suggestible'), + $search_field->get_column('sort'), + $search_field->get_column('marc_type'), + $search_field->get_column('marc_field'), + ); } } diff --git a/Koha/Objects.pm b/Koha/Objects.pm index 8ef6390..5f18b4d 100644 --- a/Koha/Objects.pm +++ b/Koha/Objects.pm @@ -91,6 +91,24 @@ sub find { return $object; } +=head3 Koha::Objects->find_or_create(); + +my $object = Koha::Objects->find_or_create( $attrs ); + +=cut + +sub find_or_create { + my ( $self, $params ) = @_; + + my $result = $self->_resultset->find_or_create($params); + + return unless $result; + + my $object = $self->object_class->_new_from_dbic($result); + + return $object; +} + =head3 Koha::Objects->search(); my @objects = Koha::Objects->search($params); diff --git a/Koha/SearchField.pm b/Koha/SearchField.pm index 72eb9c0..98c3252 100644 --- a/Koha/SearchField.pm +++ b/Koha/SearchField.pm @@ -33,6 +33,11 @@ Koha::SearchField - Koha SearchField Object class =cut +sub add_to_search_marc_maps { + my ( $self, $search_field, $params ) = @_; + return $self->_result()->add_to_search_marc_maps($search_field->_result, $params); +} + =head3 type =cut diff --git a/installer/data/mysql/atomicupdate/elastic_search.perl b/installer/data/mysql/atomicupdate/elastic_search.perl new file mode 100644 index 0000000..e8bc20e --- /dev/null +++ b/installer/data/mysql/atomicupdate/elastic_search.perl @@ -0,0 +1,83 @@ +#!/usr/bin/perl + +use Modern::Perl; + +use C4::Context; +use YAML::Syck; + +use Koha::SearchMarcMaps; +use Koha::SearchFields; + +my $dbh = C4::Context->dbh; + +$dbh->do(q|DROP TABLE IF EXISTS search_marc_to_field|); +$dbh->do(q|DROP TABLE IF EXISTS search_marc_map|); +$dbh->do(q|DROP TABLE IF EXISTS search_field|); + +# This specifies the fields that will be stored in the search engine. +$dbh->do(q| + CREATE TABLE `search_field` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine', + `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display', + `type` ENUM('string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine', + PRIMARY KEY (`id`), + UNIQUE KEY (`name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +|); + +# This contains a MARC field specifier for a given index, marc type, and marc +# field. +$dbh->do(q| + CREATE TABLE `search_marc_map` ( + id int(11) NOT NULL AUTO_INCREMENT, + index_name ENUM('biblios','authorities') NOT NULL COMMENT 'what storage index this map is for', + marc_type ENUM('marc21', 'unimarc', 'normarc') NOT NULL COMMENT 'what MARC type this map is for', + marc_field VARCHAR(255) NOT NULL COMMENT 'the MARC specifier for this field', + PRIMARY KEY(`id`), + unique key( index_name, marc_field, marc_type), + INDEX (`index_name`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +|); + +# This joins the two search tables together. We can have any combination: +# one marc field could have many search fields (maybe you want one value +# to go to 'author' and 'corporate-author) and many marc fields could go +# to one search field (e.g. all the various author fields going into +# 'author'.) +# +# a note about the sort field: +# * if all the entries for a mapping are 'null', nothing special is done with that mapping. +# * if any of the entries are not null, then a __sort field is created in ES for this mapping. In this case: +# * any mapping with sort == false WILL NOT get copied into a __sort field +# * any mapping with sort == true or is null WILL get copied into a __sort field +# * any sorts on the field name will be applied to $fieldname.'__sort' instead. +# this means that we can have search for author that includes 1xx, 245$c, and 7xx, but the sort only applies to 1xx. +$dbh->do(q| + CREATE TABLE `search_marc_to_field` ( + search_marc_map_id int(11) NOT NULL, + search_field_id int(11) NOT NULL, + facet boolean DEFAULT FALSE COMMENT 'true if a facet field should be generated for this', + suggestible boolean DEFAULT FALSE COMMENT 'true if this field can be used to generate suggestions for browse', + sort boolean DEFAULT NULL COMMENT 'true/false creates special sort handling, null doesn''t', + PRIMARY KEY(search_marc_map_id, search_field_id), + FOREIGN KEY(search_marc_map_id) REFERENCES search_marc_map(id) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY(search_field_id) REFERENCES search_field(id) ON DELETE CASCADE ON UPDATE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci +|); + +my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; +my $indexes = LoadFile( $mappings_yaml ); + +while ( my ( $index_name, $fields ) = each %$indexes ) { + while ( my ( $field_name, $data ) = each %$fields ) { + my $field_type = $data->{type}; + my $field_label = $data->{label}; + my $mappings = $data->{mappings}; + my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); + for my $mapping ( @$mappings ) { + my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); + $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } ); + } + } +} diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index c41e68e..8de3441 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -10945,39 +10945,6 @@ if ( CheckVersion($DBversion) ) { SetVersion($DBversion); } -$DBversion = "XXX"; -if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { - my $installer = C4::Installer->new(); - my $full_path = C4::Context->config('intranetdir') . "/installer/data/$installer->{dbms}/elasticsearch_mapping.sql"; - my $error = $installer->load_sql($full_path); - warn $error if $error; - print "Upgrade to $DBversion done (Bug 12478 - set up elasticsearch tables)\n"; - SetVersion($DBversion); -} - -$DBversion = "XXX"; -if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { - $dbh->do(q| - ALTER TABLE search_marc_to_field DROP FOREIGN KEY search_marc_to_field_ibfk_1 - |); - $dbh->do(q| - ALTER TABLE search_marc_to_field - ADD CONSTRAINT search_marc_to_field_ibfk_1 FOREIGN KEY (`search_marc_map_id`) REFERENCES `search_marc_map` (`id`) ON DELETE CASCADE ON UPDATE CASCADE - |); - $dbh->do(q| - ALTER TABLE search_marc_map DROP KEY index_name_2 - |); - $dbh->do(q| - ALTER TABLE search_field ADD COLUMN label VARCHAR(255) AFTER name - |); - $dbh->do(q| - UPDATE search_field SET label=name - |); - - print "Upgrade to $DBversion done (Bug 12478/14899 - DB Changes to the elasticsearch tables)\n"; - SetVersion($DBversion); -} - # DEVELOPER PROCESS, search for anything to execute in the db_update directory # SEE bug 13068 # if there is anything in the atomicupdate, read and execute it. -- 2.1.0