View | Details | Raw Unified | Return to bug 20248
Collapse All | Expand All

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +24 lines)
Lines 139-144 sub delete_index_background { Link Here
139
    $self->delete_index(@_);
139
    $self->delete_index(@_);
140
}
140
}
141
141
142
=head2 $indexer->create_index();
143
144
Create an index on the Elasticsearch server.
145
146
=cut
147
148
sub create_index {
149
    my ($self) = @_;
150
151
    if (!$self->store) {
152
        my $params  = $self->get_elasticsearch_params();
153
        $self->store(
154
            Catmandu::Store::ElasticSearch->new(
155
                %$params,
156
                index_settings => $self->get_elasticsearch_settings(),
157
                index_mappings => $self->get_elasticsearch_mappings(),
158
            )
159
        );
160
    }
161
    $self->store->bag->commit;
162
}
163
142
=head2 $indexer->drop_index();
164
=head2 $indexer->drop_index();
143
165
144
Drops the index from the elasticsearch server. Calling C<update_index>
166
Drops the index from the elasticsearch server. Calling C<update_index>
Lines 161-168 sub drop_index { Link Here
161
            )
183
            )
162
        );
184
        );
163
    }
185
    }
164
    $self->store->drop();
186
    my $store = $self->store;
165
    $self->store(undef);
187
    $self->store(undef);
188
    $store->drop();
166
}
189
}
167
190
168
sub _sanitise_records {
191
sub _sanitise_records {
(-)a/admin/searchengine/elasticsearch/mappings.pl (-7 / +10 lines)
Lines 97-111 if ( $op eq 'edit' ) { Link Here
97
        $schema->storage->txn_commit;
97
        $schema->storage->txn_commit;
98
    }
98
    }
99
}
99
}
100
elsif( $op eq 'reset' ) {
100
elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) {
101
    # TODO Move this feature to the interface
101
    Koha::SearchMarcMaps->delete;
102
    my $sure = $input->param('i_know_what_i_am_doing');
102
    Koha::SearchFields->delete;
103
    if ( $sure ) {
103
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
104
        Koha::SearchMarcMaps->search->delete;
104
    push @messages, { type => 'message', code => 'success_on_reset' };
105
        Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
105
}
106
    }
106
elsif( $op eq 'reset_confirm' ) {
107
    $template->param( reset_confirm => 1 );
107
}
108
}
108
109
110
109
my @indexes;
111
my @indexes;
110
112
111
for my $index_name (qw| biblios authorities |) {
113
for my $index_name (qw| biblios authorities |) {
Lines 114-119 for my $index_name (qw| biblios authorities |) { Link Here
114
        {   join => { search_marc_to_fields => 'search_marc_map' },
116
        {   join => { search_marc_to_fields => 'search_marc_map' },
115
            '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
117
            '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
116
            '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
118
            '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
119
            order_by => { -asc => [qw/name marc_field/] }
117
        }
120
        }
118
    );
121
    );
119
122
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (-3 / +22 lines)
Lines 79-85 a.add, a.delete { Link Here
79
          An error occurred when deleting the existing mappings. Nothing has been changed!
79
          An error occurred when deleting the existing mappings. Nothing has been changed!
80
          (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)
80
          (search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)
81
        [% CASE 'success_on_update' %]
81
        [% CASE 'success_on_update' %]
82
          Mapping updated successfully.
82
          Mappings updated successfully.
83
        [% CASE 'success_on_reset' %]
84
          Mappings have been reset successfully.
83
        [% CASE %]
85
        [% CASE %]
84
          [% m.code %]
86
          [% m.code %]
85
        [% END %]
87
        [% END %]
Lines 88-94 a.add, a.delete { Link Here
88
90
89
    <h1>Search engine configuration</h1>
91
    <h1>Search engine configuration</h1>
90
    <div class="warning">
92
    <div class="warning">
91
        Warning: Any modification in these configurations will need a total reindexation to be fully taken into account !
93
        Warning: Any modification of this configuration requires a complete index rebuild to be fully taken into account!
92
    </div>
94
    </div>
93
    [% IF errors %]
95
    [% IF errors %]
94
        <div class="error">
96
        <div class="error">
Lines 107-112 a.add, a.delete { Link Here
107
        </div>
109
        </div>
108
    [% END %]
110
    [% END %]
109
111
112
    [% IF reset_confirm %]
113
        <div class="dialog alert">
114
            <h3>The current mappings you seen on the screen will be erased and replaced by the mappings in the mappings.yaml file.</h3>
115
            <form action="?" method="post">
116
                <input type="hidden" name="op" value="reset_confirmed" />
117
                <button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, reset mappings</button>
118
            </form>
119
            <br>
120
            <form action="?" method="post">
121
                <button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not reset mappings</button>
122
            </form>
123
        </div>
124
    [% ELSE %]
110
    <form method="post">
125
    <form method="post">
111
        <input type="hidden" name="op" value="edit" />
126
        <input type="hidden" name="op" value="edit" />
112
        <div id="tabs" class="toptabs" style="clear:both">
127
        <div id="tabs" class="toptabs" style="clear:both">
Lines 286-293 a.add, a.delete { Link Here
286
                </div>
301
                </div>
287
            [% END %]
302
            [% END %]
288
        </div>
303
        </div>
289
        <p><button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button></p>
304
        <p>
305
            <button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button>
306
            <a class="btn btn-default" href="?op=reset_confirm"><i class="fa fa-refresh" aria-hidden="true"></i> Reset Mappings</a>
307
        </p>
290
    </form>
308
    </form>
309
    [% END %]
291
</div>
310
</div>
292
311
293
</div>
312
</div>
(-)a/misc/search_tools/rebuild_elastic_search.pl (-8 / +12 lines)
Lines 92-105 use MARC::Record; Link Here
92
use Modern::Perl;
92
use Modern::Perl;
93
use Pod::Usage;
93
use Pod::Usage;
94
94
95
use Data::Dumper; # TODO remove
96
97
my $verbose = 0;
95
my $verbose = 0;
98
my $commit = 5000;
96
my $commit = 5000;
99
my ($delete, $help, $man);
97
my ($delete, $help, $man);
100
my ($index_biblios, $index_authorities);
98
my ($index_biblios, $index_authorities);
101
my (@biblionumbers);
99
my (@biblionumbers);
102
100
101
$|=1; # flushes output
102
103
GetOptions(
103
GetOptions(
104
    'c|commit=i'       => \$commit,
104
    'c|commit=i'       => \$commit,
105
    'd|delete'         => \$delete,
105
    'd|delete'         => \$delete,
Lines 161-170 sub do_reindex { Link Here
161
161
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
162
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163
    if ($delete) {
163
    if ($delete) {
164
165
        # We know it's safe to not recreate the indexer because update_index
166
        # hasn't been called yet.
167
        $indexer->drop_index();
164
        $indexer->drop_index();
165
        $indexer->create_index();
168
    }
166
    }
169
167
170
    my $count        = 0;
168
    my $count        = 0;
Lines 173-195 sub do_reindex { Link Here
173
    while ( my $record = $next->() ) {
171
    while ( my $record = $next->() ) {
174
        my $id     = $record->id;
172
        my $id     = $record->id;
175
        my $record = $record->record;
173
        my $record = $record->record;
176
        _log( 1, "$id\n" );
177
        $count++;
174
        $count++;
175
        if ( $verbose == 1 ) {
176
            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
177
        } else {
178
            _log( 2, "$id\n" );
179
        }
178
180
179
        push @id_buffer,     $id;
181
        push @id_buffer,     $id;
180
        push @commit_buffer, $record;
182
        push @commit_buffer, $record;
181
        if ( !( --$commit_count ) ) {
183
        if ( !( --$commit_count ) ) {
182
            _log( 2, "Committing...\n" );
184
            _log( 1, "Committing $commit records..." );
183
            $indexer->update_index( \@id_buffer, \@commit_buffer );
185
            $indexer->update_index( \@id_buffer, \@commit_buffer );
184
            $commit_count  = $commit;
186
            $commit_count  = $commit;
185
            @id_buffer     = ();
187
            @id_buffer     = ();
186
            @commit_buffer = ();
188
            @commit_buffer = ();
189
            _log( 1, " done\n" );
187
        }
190
        }
188
    }
191
    }
189
192
190
    # There are probably uncommitted records
193
    # There are probably uncommitted records
194
    _log( 1, "Committing final records...\n" );
191
    $indexer->update_index( \@id_buffer, \@commit_buffer );
195
    $indexer->update_index( \@id_buffer, \@commit_buffer );
192
    _log( 1, "$count records indexed.\n" );
196
    _log( 1, "Total $count records indexed\n" );
193
}
197
}
194
198
195
# Checks some basic stuff to ensure that it's sane before we start.
199
# Checks some basic stuff to ensure that it's sane before we start.
(-)a/t/db_dependent/Koha_Elasticsearch_Indexer.t (-3 / +32 lines)
Lines 17-23 Link Here
17
17
18
use Modern::Perl;
18
use Modern::Perl;
19
19
20
use Test::More tests => 6;
20
use Test::More tests => 7;
21
use Test::MockModule;
21
use Test::MockModule;
22
use t::lib::Mocks;
22
use t::lib::Mocks;
23
23
Lines 52-63 SKIP: { Link Here
52
52
53
    eval { $indexer->get_elasticsearch_params; };
53
    eval { $indexer->get_elasticsearch_params; };
54
54
55
    skip 'ElasticSeatch configuration not available', 1
55
    skip 'Elasticsearch configuration not available', 1
56
        if $@;
56
        if $@;
57
57
58
    ok( $indexer->update_index(undef,$records), 'Update Index' );
58
    ok( $indexer->update_index(undef,$records), 'Update Index' );
59
}
59
}
60
60
61
subtest 'create_index() tests' => sub {
62
63
    plan tests => 3;
64
65
    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
66
    $se->mock( 'get_elasticsearch_params', sub {
67
        my ($self, $sub ) = @_;
68
69
        my $method = $se->original( 'get_elasticsearch_params' );
70
        my $params = $method->( $self );
71
        $params->{index_name} .= '__test';
72
        return $params;
73
    });
74
75
    my $indexer;
76
    ok(
77
        $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
78
        'Creating a new indexer object'
79
    );
80
    ok(
81
        $indexer->create_index(),
82
        'Creating an index'
83
    );
84
    $indexer->drop_index();
85
    ok(
86
        $indexer->drop_index(),
87
        'Dropping the index'
88
    );
89
};
90
61
subtest '_convert_marc_to_json() tests' => sub {
91
subtest '_convert_marc_to_json() tests' => sub {
62
92
63
    plan tests => 2;
93
    plan tests => 2;
64
- 

Return to bug 20248