Attachment #72861 for bug #20248

View | Details | Raw Unified | Return to bug 20248
Collapse All | Expand All




    $self->delete_index(@_);
}

=head2 $indexer->create_index();

Create an index on the Elasticsearch server.

=cut

sub create_index {
    my ($self) = @_;

    if (!$self->store) {
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
            )
        );
    }
    $self->store->bag->commit;
}

=head2 $indexer->drop_index();

Drops the index from the elasticsearch server. Calling C<update_index>

            )
        );
    }
    my $store = $self->store;
    $self->store(undef);
    $store->drop();
}

sub _sanitise_records {




        $schema->storage->txn_commit;
    }
}
elsif( $op eq 'reset' || $op eq 'reset_confirmed' ) {
    Koha::SearchMarcMaps->delete;
    Koha::SearchFields->delete;
    Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
    push @messages, { type => 'message', code => 'success_on_reset' };
}
elsif( $op eq 'reset_confirm' ) {
    $template->param( reset_confirm => 1 );
}


my @indexes;

for my $index_name (qw| biblios authorities |) {

        {   join => { search_marc_to_fields => 'search_marc_map' },
            '+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
            '+as'     => [ 'facet',                       'suggestible',                       'sort',                       'marc_field' ],
            order_by => { -asc => [qw/name marc_field/] }
        }
    );


Lines 79-85 a.add, a.delete { Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt (-3 / +22 lines)
79	An error occurred when deleting the existing mappings. Nothing has been changed!	79	An error occurred when deleting the existing mappings. Nothing has been changed!
80	(search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)	80	(search field [% m.values.field_name %] with mapping [% m.values.marc_field %].)
81	[% CASE 'success_on_update' %]	81	[% CASE 'success_on_update' %]
82	Mapping updated successfully.	82	Mappings updated successfully.
		83	[% CASE 'success_on_reset' %]
		84	Mappings have been reset successfully.
83	[% CASE %]	85	[% CASE %]
84	[% m.code %]	86	[% m.code %]
85	[% END %]	87	[% END %]
Lines 88-94 a.add, a.delete { Link Here
88		90
89	<h1>Search engine configuration</h1>	91	<h1>Search engine configuration</h1>
90	<div class="warning">	92	<div class="warning">
91	Warning: Any modification in these configurations will need a total reindexation to be fully taken into account !	93	Warning: Any modification of this configuration requires a complete index rebuild to be fully taken into account!
92	</div>	94	</div>
93	[% IF errors %]	95	[% IF errors %]
94	<div class="error">	96	<div class="error">
Lines 107-112 a.add, a.delete { Link Here
107	</div>	109	</div>
108	[% END %]	110	[% END %]
109		111
		112	[% IF reset_confirm %]
		113	<div class="dialog alert">
		114	<h3>The current mappings you seen on the screen will be erased and replaced by the mappings in the mappings.yaml file.</h3>
		115	<form action="?" method="post">
		116	<input type="hidden" name="op" value="reset_confirmed" />
		117	<button type="submit" class="approve"><i class="fa fa-fw fa-check"></i> Yes, reset mappings</button>
		118	</form>
		119	<br>
		120	<form action="?" method="post">
		121	<button type="submit" class="deny"><i class="fa fa-fw fa-remove"></i> No, do not reset mappings</button>
		122	</form>
		123	</div>
		124	[% ELSE %]
110	<form method="post">	125	<form method="post">
111	<input type="hidden" name="op" value="edit" />	126	<input type="hidden" name="op" value="edit" />
112	<div id="tabs" class="toptabs" style="clear:both">	127	<div id="tabs" class="toptabs" style="clear:both">
Lines 286-293 a.add, a.delete { Link Here
286	</div>	301	</div>
287	[% END %]	302	[% END %]
288	</div>	303	</div>
289	<p><button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button></p>	304	<p>
		305	<button class="btn btn-default" type="submit"><i class="fa fa-hdd-o" aria-hidden="true"></i> Save</button>
		306	<a class="btn btn-default" href="?op=reset_confirm"><i class="fa fa-refresh" aria-hidden="true"></i> Reset Mappings</a>
		307	</p>
290	</form>	308	</form>
		309	[% END %]
291	</div>	310	</div>
292		311
293	</div>	312	</div>




use Modern::Perl;
use Pod::Usage;

use Data::Dumper; # TODO remove

my $verbose = 0;
my $commit = 5000;
my ($delete, $help, $man);
my ($index_biblios, $index_authorities);
my (@biblionumbers);

$|=1; # flushes output

GetOptions(
    'c|commit=i'       => \$commit,
    'd|delete'         => \$delete,


    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
    if ($delete) {

        # We know it's safe to not recreate the indexer because update_index
        # hasn't been called yet.
        $indexer->drop_index();
        $indexer->create_index();
    }

    my $count        = 0;

    while ( my $record = $next->() ) {
        my $id     = $record->id;
        my $record = $record->record;
        _log( 1, "$id\n" );
        $count++;
        if ( $verbose == 1 ) {
            _log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
        } else {
            _log( 2, "$id\n" );
        }

        push @id_buffer,     $id;
        push @commit_buffer, $record;
        if ( !( --$commit_count ) ) {
            _log( 1, "Committing $commit records..." );
            $indexer->update_index( \@id_buffer, \@commit_buffer );
            $commit_count  = $commit;
            @id_buffer     = ();
            @commit_buffer = ();
            _log( 1, " done\n" );
        }
    }

    # There are probably uncommitted records
    _log( 1, "Committing final records...\n" );
    $indexer->update_index( \@id_buffer, \@commit_buffer );
    _log( 1, "Total $count records indexed\n" );
}

# Checks some basic stuff to ensure that it's sane before we start.





use Modern::Perl;

use Test::More tests => 7;
use Test::MockModule;
use t::lib::Mocks;



    eval { $indexer->get_elasticsearch_params; };

    skip 'Elasticsearch configuration not available', 1
        if $@;

    ok( $indexer->update_index(undef,$records), 'Update Index' );
}

subtest 'create_index() tests' => sub {

    plan tests => 3;

    my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
    $se->mock( 'get_elasticsearch_params', sub {
        my ($self, $sub ) = @_;

        my $method = $se->original( 'get_elasticsearch_params' );
        my $params = $method->( $self );
        $params->{index_name} .= '__test';
        return $params;
    });

    my $indexer;
    ok(
        $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
        'Creating a new indexer object'
    );
    ok(
        $indexer->create_index(),
        'Creating an index'
    );
    $indexer->drop_index();
    ok(
        $indexer->drop_index(),
        'Dropping the index'
    );
};

subtest '_convert_marc_to_json() tests' => sub {

    plan tests => 2;
- 

Return to bug 20248

Lines 139-144 sub delete_index_background { Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-1 / +24 lines)
139	$self->delete_index(@_);	139	$self->delete_index(@_);
140	}	140	}
141		141
		142	=head2 $indexer->create_index();
		143
		144	Create an index on the Elasticsearch server.
		145
		146	=cut
		147
		148	sub create_index {
		149	my ($self) = @_;
		150
		151	if (!$self->store) {
		152	my $params = $self->get_elasticsearch_params();
		153	$self->store(
		154	Catmandu::Store::ElasticSearch->new(
		155	%$params,
		156	index_settings => $self->get_elasticsearch_settings(),
		157	index_mappings => $self->get_elasticsearch_mappings(),
		158	)
		159	);
		160	}
		161	$self->store->bag->commit;
		162	}
		163
142	=head2 $indexer->drop_index();	164	=head2 $indexer->drop_index();
143		165
144	Drops the index from the elasticsearch server. Calling C<update_index>	166	Drops the index from the elasticsearch server. Calling C<update_index>
Lines 161-168 sub drop_index { Link Here
161	)	183	)
162	);	184	);
163	}	185	}
164	$self->store->drop();	186	my $store = $self->store;
165	$self->store(undef);	187	$self->store(undef);
		188	$store->drop();
166	}	189	}
167		190
168	sub _sanitise_records {	191	sub _sanitise_records {

Lines 92-105 use MARC::Record; Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-8 / +12 lines)
92	use Modern::Perl;	92	use Modern::Perl;
93	use Pod::Usage;	93	use Pod::Usage;
94		94
95	use Data::Dumper; # TODO remove
96
97	my $verbose = 0;	95	my $verbose = 0;
98	my $commit = 5000;	96	my $commit = 5000;
99	my ($delete, $help, $man);	97	my ($delete, $help, $man);
100	my ($index_biblios, $index_authorities);	98	my ($index_biblios, $index_authorities);
101	my (@biblionumbers);	99	my (@biblionumbers);
102		100
		101	$\|=1; # flushes output
		102
103	GetOptions(	103	GetOptions(
104	'c\|commit=i' => \$commit,	104	'c\|commit=i' => \$commit,
105	'd\|delete' => \$delete,	105	'd\|delete' => \$delete,
Lines 161-170 sub do_reindex { Link Here
161		161
162	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );	162	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163	if ($delete) {	163	if ($delete) {
164
165	# We know it's safe to not recreate the indexer because update_index
166	# hasn't been called yet.
167	$indexer->drop_index();	164	$indexer->drop_index();
		165	$indexer->create_index();
168	}	166	}
169		167
170	my $count = 0;	168	my $count = 0;
Lines 173-195 sub do_reindex { Link Here
173	while ( my $record = $next->() ) {	171	while ( my $record = $next->() ) {
174	my $id = $record->id;	172	my $id = $record->id;
175	my $record = $record->record;	173	my $record = $record->record;
176	_log( 1, "$id\n" );
177	$count++;	174	$count++;
		175	if ( $verbose == 1 ) {
		176	_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);
		177	} else {
		178	_log( 2, "$id\n" );
		179	}
178		180
179	push @id_buffer, $id;	181	push @id_buffer, $id;
180	push @commit_buffer, $record;	182	push @commit_buffer, $record;
181	if ( !( --$commit_count ) ) {	183	if ( !( --$commit_count ) ) {
182	_log( 2, "Committing...\n" );	184	_log( 1, "Committing $commit records..." );
183	$indexer->update_index( \@id_buffer, \@commit_buffer );	185	$indexer->update_index( \@id_buffer, \@commit_buffer );
184	$commit_count = $commit;	186	$commit_count = $commit;
185	@id_buffer = ();	187	@id_buffer = ();
186	@commit_buffer = ();	188	@commit_buffer = ();
		189	_log( 1, " done\n" );
187	}	190	}
188	}	191	}
189		192
190	# There are probably uncommitted records	193	# There are probably uncommitted records
		194	_log( 1, "Committing final records...\n" );
191	$indexer->update_index( \@id_buffer, \@commit_buffer );	195	$indexer->update_index( \@id_buffer, \@commit_buffer );
192	_log( 1, "$count records indexed.\n" );	196	_log( 1, "Total $count records indexed\n" );
193	}	197	}
194		198
195	# Checks some basic stuff to ensure that it's sane before we start.	199	# Checks some basic stuff to ensure that it's sane before we start.

Lines 97-111 if ( $op eq 'edit' ) { Link Here

(-)a/admin/searchengine/elasticsearch/mappings.pl (-7 / +10 lines)
97	$schema->storage->txn_commit;	97	$schema->storage->txn_commit;
98	}	98	}
99	}	99	}
100	elsif( $op eq 'reset' ) {	100	elsif( $op eq 'reset' \|\| $op eq 'reset_confirmed' ) {
101	# TODO Move this feature to the interface	101	Koha::SearchMarcMaps->delete;
102	my $sure = $input->param('i_know_what_i_am_doing');	102	Koha::SearchFields->delete;
103	if ( $sure ) {	103	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
104	Koha::SearchMarcMaps->search->delete;	104	push @messages, { type => 'message', code => 'success_on_reset' };
105	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;	105	}
106	}	106	elsif( $op eq 'reset_confirm' ) {
		107	$template->param( reset_confirm => 1 );
107	}	108	}
108		109
		110
109	my @indexes;	111	my @indexes;
110		112
111	for my $index_name (qw\| biblios authorities \|) {	113	for my $index_name (qw\| biblios authorities \|) {
Lines 114-119 for my $index_name (qw\| biblios authorities \|) { Link Here
114	{ join => { search_marc_to_fields => 'search_marc_map' },	116	{ join => { search_marc_to_fields => 'search_marc_map' },
115	'+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],	117	'+select' => [ 'search_marc_to_fields.facet', 'search_marc_to_fields.suggestible', 'search_marc_to_fields.sort', 'search_marc_map.marc_field' ],
116	'+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ],	118	'+as' => [ 'facet', 'suggestible', 'sort', 'marc_field' ],
		119	order_by => { -asc => [qw/name marc_field/] }
117	}	120	}
118	);	121	);
119		122

Lines 17-23 Link Here

(-)a/t/db_dependent/Koha_Elasticsearch_Indexer.t (-3 / +32 lines)
17		17
18	use Modern::Perl;	18	use Modern::Perl;
19		19
20	use Test::More tests => 6;	20	use Test::More tests => 7;
21	use Test::MockModule;	21	use Test::MockModule;
22	use t::lib::Mocks;	22	use t::lib::Mocks;
23		23
Lines 52-63 SKIP: { Link Here
52		52
53	eval { $indexer->get_elasticsearch_params; };	53	eval { $indexer->get_elasticsearch_params; };
54		54
55	skip 'ElasticSeatch configuration not available', 1	55	skip 'Elasticsearch configuration not available', 1
56	if $@;	56	if $@;
57		57
58	ok( $indexer->update_index(undef,$records), 'Update Index' );	58	ok( $indexer->update_index(undef,$records), 'Update Index' );
59	}	59	}
60		60
		61	subtest 'create_index() tests' => sub {
		62
		63	plan tests => 3;
		64
		65	my $se = Test::MockModule->new( 'Koha::SearchEngine::Elasticsearch' );
		66	$se->mock( 'get_elasticsearch_params', sub {
		67	my ($self, $sub ) = @_;
		68
		69	my $method = $se->original( 'get_elasticsearch_params' );
		70	my $params = $method->( $self );
		71	$params->{index_name} .= '__test';
		72	return $params;
		73	});
		74
		75	my $indexer;
		76	ok(
		77	$indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ 'index' => 'biblios' }),
		78	'Creating a new indexer object'
		79	);
		80	ok(
		81	$indexer->create_index(),
		82	'Creating an index'
		83	);
		84	$indexer->drop_index();
		85	ok(
		86	$indexer->drop_index(),
		87	'Dropping the index'
		88	);
		89	};
		90
61	subtest '_convert_marc_to_json() tests' => sub {	91	subtest '_convert_marc_to_json() tests' => sub {
62		92
63	plan tests => 2;	93	plan tests => 2;
64	-