Bugzilla – Attachment 43348 Details for
Bug 12478
Elasticsearch support for Koha
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 12478: Move mapping attributes to the join table
Bug-12478-Move-mapping-attributes-to-the-join-tabl.patch (text/plain), 10.30 KB, created by
Jonathan Druart
on 2015-10-12 16:18:09 UTC
(
hide
)
Description:
Bug 12478: Move mapping attributes to the join table
Filename:
MIME Type:
Creator:
Jonathan Druart
Created:
2015-10-12 16:18:09 UTC
Size:
10.30 KB
patch
obsolete
>From 1976be11ad238afd6db98f700379a2df87a977db Mon Sep 17 00:00:00 2001 >From: Jonathan Druart <jonathan.druart@bugs.koha-community.org> >Date: Mon, 12 Oct 2015 16:46:16 +0100 >Subject: [PATCH] Bug 12478: Move mapping attributes to the join table > >To make things consistent, the attributes of the mappings (sort, >suggestible, facet) should be moved to the join tables >(search_marc_to_field). >Indeed the search_marc_map could represent a mapping for several search >fields with different attributes. Before this change it was not >possible. >--- > Koha/ElasticSearch.pm | 53 ++++++++------ > Koha/Objects.pm | 18 +++++ > Koha/SearchField.pm | 5 ++ > .../data/mysql/atomicupdate/elastic_search.perl | 83 ++++++++++++++++++++++ > installer/data/mysql/updatedatabase.pl | 33 --------- > 5 files changed, 137 insertions(+), 55 deletions(-) > create mode 100644 installer/data/mysql/atomicupdate/elastic_search.perl > >diff --git a/Koha/ElasticSearch.pm b/Koha/ElasticSearch.pm >index e91d5b8..d56c344 100644 >--- a/Koha/ElasticSearch.pm >+++ b/Koha/ElasticSearch.pm >@@ -374,29 +374,38 @@ sub _foreach_mapping { > my ( $self, $sub ) = @_; > > # TODO use a caching framework here >- my $database = Koha::Database->new(); >- my $schema = $database->schema(); >- my $rs = >- $schema->resultset('SearchMarcMap') >- ->search( { index_name => $self->index } ); >- for my $row ( $rs->all ) { >- my $marc_type = $row->marc_type; >- my $marc_field = $row->marc_field; >- my $facet = $row->facet; >- my $suggestible = $row->suggestible; >- my $search_field = $row->search_fields(); >- my $sort = $row->sort(); >- for my $sf ( $search_field->all ) { >- $sub->( >- $sf->name, >- $sf->type, >- $facet, >- $suggestible, >- $sort, >- $marc_type, >- $marc_field, >- ); >+ my $search_fields = Koha::Database->schema->resultset('SearchField')->search( >+ { >+ 'search_marc_map.index_name' => $self->index, >+ }, >+ { join => { search_marc_to_fields => 'search_marc_map' }, >+ '+select' => [ >+ 'search_marc_to_fields.facet', >+ 'search_marc_to_fields.suggestible', >+ 'search_marc_to_fields.sort', >+ 'search_marc_map.marc_type', >+ 'search_marc_map.marc_field', >+ ], >+ '+as' => [ >+ 'facet', >+ 'suggestible', >+ 'sort', >+ 'marc_type', >+ 'marc_field', >+ ], > } >+ ); >+ >+ while ( my $search_field = $search_fields->next ) { >+ $sub->( >+ $search_field->name, >+ $search_field->type, >+ $search_field->get_column('facet'), >+ $search_field->get_column('suggestible'), >+ $search_field->get_column('sort'), >+ $search_field->get_column('marc_type'), >+ $search_field->get_column('marc_field'), >+ ); > } > } > >diff --git a/Koha/Objects.pm b/Koha/Objects.pm >index 8ef6390..5f18b4d 100644 >--- a/Koha/Objects.pm >+++ b/Koha/Objects.pm >@@ -91,6 +91,24 @@ sub find { > return $object; > } > >+=head3 Koha::Objects->find_or_create(); >+ >+my $object = Koha::Objects->find_or_create( $attrs ); >+ >+=cut >+ >+sub find_or_create { >+ my ( $self, $params ) = @_; >+ >+ my $result = $self->_resultset->find_or_create($params); >+ >+ return unless $result; >+ >+ my $object = $self->object_class->_new_from_dbic($result); >+ >+ return $object; >+} >+ > =head3 Koha::Objects->search(); > > my @objects = Koha::Objects->search($params); >diff --git a/Koha/SearchField.pm b/Koha/SearchField.pm >index 72eb9c0..98c3252 100644 >--- a/Koha/SearchField.pm >+++ b/Koha/SearchField.pm >@@ -33,6 +33,11 @@ Koha::SearchField - Koha SearchField Object class > > =cut > >+sub add_to_search_marc_maps { >+ my ( $self, $search_field, $params ) = @_; >+ return $self->_result()->add_to_search_marc_maps($search_field->_result, $params); >+} >+ > =head3 type > > =cut >diff --git a/installer/data/mysql/atomicupdate/elastic_search.perl b/installer/data/mysql/atomicupdate/elastic_search.perl >new file mode 100644 >index 0000000..e8bc20e >--- /dev/null >+++ b/installer/data/mysql/atomicupdate/elastic_search.perl >@@ -0,0 +1,83 @@ >+#!/usr/bin/perl >+ >+use Modern::Perl; >+ >+use C4::Context; >+use YAML::Syck; >+ >+use Koha::SearchMarcMaps; >+use Koha::SearchFields; >+ >+my $dbh = C4::Context->dbh; >+ >+$dbh->do(q|DROP TABLE IF EXISTS search_marc_to_field|); >+$dbh->do(q|DROP TABLE IF EXISTS search_marc_map|); >+$dbh->do(q|DROP TABLE IF EXISTS search_field|); >+ >+# This specifies the fields that will be stored in the search engine. >+$dbh->do(q| >+ CREATE TABLE `search_field` ( >+ `id` int(11) NOT NULL AUTO_INCREMENT, >+ `name` varchar(255) NOT NULL COMMENT 'the name of the field as it will be stored in the search engine', >+ `label` varchar(255) NOT NULL COMMENT 'the human readable name of the field, for display', >+ `type` ENUM('string', 'date', 'number', 'boolean', 'sum') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine', >+ PRIMARY KEY (`id`), >+ UNIQUE KEY (`name`) >+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci >+|); >+ >+# This contains a MARC field specifier for a given index, marc type, and marc >+# field. >+$dbh->do(q| >+ CREATE TABLE `search_marc_map` ( >+ id int(11) NOT NULL AUTO_INCREMENT, >+ index_name ENUM('biblios','authorities') NOT NULL COMMENT 'what storage index this map is for', >+ marc_type ENUM('marc21', 'unimarc', 'normarc') NOT NULL COMMENT 'what MARC type this map is for', >+ marc_field VARCHAR(255) NOT NULL COMMENT 'the MARC specifier for this field', >+ PRIMARY KEY(`id`), >+ unique key( index_name, marc_field, marc_type), >+ INDEX (`index_name`) >+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci >+|); >+ >+# This joins the two search tables together. We can have any combination: >+# one marc field could have many search fields (maybe you want one value >+# to go to 'author' and 'corporate-author) and many marc fields could go >+# to one search field (e.g. all the various author fields going into >+# 'author'.) >+# >+# a note about the sort field: >+# * if all the entries for a mapping are 'null', nothing special is done with that mapping. >+# * if any of the entries are not null, then a __sort field is created in ES for this mapping. In this case: >+# * any mapping with sort == false WILL NOT get copied into a __sort field >+# * any mapping with sort == true or is null WILL get copied into a __sort field >+# * any sorts on the field name will be applied to $fieldname.'__sort' instead. >+# this means that we can have search for author that includes 1xx, 245$c, and 7xx, but the sort only applies to 1xx. >+$dbh->do(q| >+ CREATE TABLE `search_marc_to_field` ( >+ search_marc_map_id int(11) NOT NULL, >+ search_field_id int(11) NOT NULL, >+ facet boolean DEFAULT FALSE COMMENT 'true if a facet field should be generated for this', >+ suggestible boolean DEFAULT FALSE COMMENT 'true if this field can be used to generate suggestions for browse', >+ sort boolean DEFAULT NULL COMMENT 'true/false creates special sort handling, null doesn''t', >+ PRIMARY KEY(search_marc_map_id, search_field_id), >+ FOREIGN KEY(search_marc_map_id) REFERENCES search_marc_map(id) ON DELETE CASCADE ON UPDATE CASCADE, >+ FOREIGN KEY(search_field_id) REFERENCES search_field(id) ON DELETE CASCADE ON UPDATE CASCADE >+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci >+|); >+ >+my $mappings_yaml = C4::Context->config('intranetdir') . '/admin/searchengine/elasticsearch/mappings.yaml'; >+my $indexes = LoadFile( $mappings_yaml ); >+ >+while ( my ( $index_name, $fields ) = each %$indexes ) { >+ while ( my ( $field_name, $data ) = each %$fields ) { >+ my $field_type = $data->{type}; >+ my $field_label = $data->{label}; >+ my $mappings = $data->{mappings}; >+ my $search_field = Koha::SearchFields->find_or_create({ name => $field_name, label => $field_label, type => $field_type }, { key => 'name' }); >+ for my $mapping ( @$mappings ) { >+ my $marc_field = Koha::SearchMarcMaps->find_or_create({ index_name => $index_name, marc_type => $mapping->{marc_type}, marc_field => $mapping->{marc_field} }); >+ $search_field->add_to_search_marc_maps($marc_field, { facet => $mapping->{facet}, suggestible => $mapping->{suggestible}, sort => $mapping->{sort} } ); >+ } >+ } >+} >diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl >index c41e68e..8de3441 100755 >--- a/installer/data/mysql/updatedatabase.pl >+++ b/installer/data/mysql/updatedatabase.pl >@@ -10945,39 +10945,6 @@ if ( CheckVersion($DBversion) ) { > SetVersion($DBversion); > } > >-$DBversion = "XXX"; >-if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { >- my $installer = C4::Installer->new(); >- my $full_path = C4::Context->config('intranetdir') . "/installer/data/$installer->{dbms}/elasticsearch_mapping.sql"; >- my $error = $installer->load_sql($full_path); >- warn $error if $error; >- print "Upgrade to $DBversion done (Bug 12478 - set up elasticsearch tables)\n"; >- SetVersion($DBversion); >-} >- >-$DBversion = "XXX"; >-if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { >- $dbh->do(q| >- ALTER TABLE search_marc_to_field DROP FOREIGN KEY search_marc_to_field_ibfk_1 >- |); >- $dbh->do(q| >- ALTER TABLE search_marc_to_field >- ADD CONSTRAINT search_marc_to_field_ibfk_1 FOREIGN KEY (`search_marc_map_id`) REFERENCES `search_marc_map` (`id`) ON DELETE CASCADE ON UPDATE CASCADE >- |); >- $dbh->do(q| >- ALTER TABLE search_marc_map DROP KEY index_name_2 >- |); >- $dbh->do(q| >- ALTER TABLE search_field ADD COLUMN label VARCHAR(255) AFTER name >- |); >- $dbh->do(q| >- UPDATE search_field SET label=name >- |); >- >- print "Upgrade to $DBversion done (Bug 12478/14899 - DB Changes to the elasticsearch tables)\n"; >- SetVersion($DBversion); >-} >- > # DEVELOPER PROCESS, search for anything to execute in the db_update directory > # SEE bug 13068 > # if there is anything in the atomicupdate, read and execute it. >-- >2.1.0
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 12478
:
29204
|
31128
|
31129
|
33136
|
33137
|
33138
|
33139
|
33140
|
33141
|
35654
|
40903
|
42030
|
42031
|
42032
|
42033
|
42062
|
42063
|
42064
|
42065
|
42066
|
42067
|
42068
|
42069
|
42392
|
42393
|
43127
|
43128
|
43129
|
43130
|
43131
|
43132
|
43133
|
43134
|
43140
|
43141
|
43142
|
43148
|
43345
|
43346
|
43347
|
43348
|
43349
|
43350
|
43351
|
43352
|
43373
|
47767
|
47768
|
50115
|
50159
|
50211
|
50212
|
50213
|
50214
|
50245
|
50246
|
50291
|
50292