Attachment #70815 for bug #18948

View | Details | Raw Unified | Return to bug 18948
Collapse All | Expand All





=head1 FUNCTIONS

=head2 Koha::SearchEngine::Elasticsearch::Indexer->({index => $index_name});

$index_name should be biblios or authorities
call our SUPER class and then set ES parameters

=cut

sub new {
    my $class = shift @_;
    my $self = $class->SUPER::new(@_);
    my $params  = $self->get_elasticsearch_params();
    $self->store(
        Catmandu::Store::ElasticSearch->new(
            %$params,
            index_settings => $self->get_elasticsearch_settings(),
            index_mappings => $self->get_elasticsearch_mappings(),
        )
    );
    return $self;
}

=head2 $indexer->update_index($biblionums, $records);

C<$biblionums> is an arrayref containing the biblionumbers for the records.

    }

    my $from    = $self->_convert_marc_to_json($records);
    if ( !$self->store ) {
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
            )
        );
    }

    print Data::Dumper::Dumper( $from->to_array );
    $self->store->bag->add_many($from);
    $self->store->bag->commit;
    return 1;

sub delete_index {
    my ($self, $biblionums) = @_;

    if ( !$self->store ) {
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
            )
        );
    }
    $self->store->bag->delete($_) foreach @$biblionums;
    $self->store->bag->commit;
}

sub drop_index {
    my ($self) = @_;

    if (!$self->store) {
        # If this index doesn't exist, this will create it. Then it'll be
        # deleted. That's not the end of the world however.
        my $params  = $self->get_elasticsearch_params();
        $self->store(
            Catmandu::Store::ElasticSearch->new(
                %$params,
                index_settings => $self->get_elasticsearch_settings(),
                index_mappings => $self->get_elasticsearch_mappings(),
            )
        );
    }
    $self->store->drop();
    my $params  = $self->get_elasticsearch_params();
    $self->store(
        Catmandu::Store::ElasticSearch->new(
            %$params,
            index_settings => $self->get_elasticsearch_settings(),
            index_mappings => $self->get_elasticsearch_mappings(),
        )
    );
}


=head2 $indexer->do_reindex();

Performs a reindex based on records passed in

=cut

sub do_reindex {
    my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;

    _log(1, $verbose, "Indexing ".$self->index."\n");

    if ( scalar @$biblionumbers ) {
        if ( $self->index eq 'biblios' ) {
            $next = sub {
                my $r = shift @$biblionumbers;
                return () unless defined $r;
                return ( $r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
            };
        }
        elsif ( $self->index eq 'authorities' ) {
            $next = sub {
                my $r = shift @$biblionumbers;
                return () unless defined $r;
                my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
                return ($r, $a->record);
            };
        }
    }
    else {
        my $records = $self->index eq 'biblios' ?
            Koha::BiblioUtils->get_all_biblios_iterator() : $self->index eq 'authorities' ?
                Koha::MetadataRecord::Authority->get_all_authorities_iterator() : die "Reindex called on undefined index";
        $next = sub { $records->next(); };
    }

warn
    $self->drop_index() if ( $delete );

    my $count        = 0;
    my $commit_count = $commit;
    my ( @id_buffer, @commit_buffer );
    while ( my $record = $next->() ) {
        my $id     = $record->id;
        my $record = $record->record;
        _log( 1, $verbose, "$id\n" );
        $count++;

        push @id_buffer,     $id;
        push @commit_buffer, $record;
        if ( !( --$commit_count ) ) {
            _log( 2, $verbose, "Committing...\n" );
            $self->update_index( \@id_buffer, \@commit_buffer );
            $commit_count  = $commit;
            @id_buffer     = ();
            @commit_buffer = ();
        }
    }
    # There are probably uncommitted records (beacuse we didn't hit exact commit count)
    $self->update_index( \@id_buffer, \@commit_buffer );
    _log( 1, $verbose, "$count records indexed.\n" );
}


sub _log {
    my ($level, $verbose, $msg) = @_;
    print $msg if ($verbose >= $level);
}

sub _sanitise_records {




sanity_check();

my $next;
print "$verbose verbosity";
if ($index_biblios) {
    my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX;
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
    $indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers);
            my $r = shift @biblionumbers;
            return () unless defined $r;
            return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
        };
    } else {
        my $records = Koha::BiblioUtils->get_all_biblios_iterator();
        $next = sub {
            $records->next();
        }
    }
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
}
if ($index_authorities) {
    my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX;
    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
    $indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers);
            my $r = shift @biblionumbers;
            return () unless defined $r;
            my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
            return ($r, $a->record);
        };
    } else {
        my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
        $next = sub {
            $records->next();
        }
    }
    do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
}

sub do_reindex {
    my ( $next, $index_name ) = @_;

    my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
    if ($delete) {

        # We know it's safe to not recreate the indexer because update_index
        # hasn't been called yet.
        $indexer->drop_index();
    }

    my $count        = 0;
    my $commit_count = $commit;
    my ( @id_buffer, @commit_buffer );
    while ( my $record = $next->() ) {
        my $id     = $record->id;
        my $record = $record->record;
        _log( 1, "$id\n" );
        $count++;

        push @id_buffer,     $id;
        push @commit_buffer, $record;
        if ( !( --$commit_count ) ) {
            _log( 2, "Committing...\n" );
            $indexer->update_index( \@id_buffer, \@commit_buffer );
            $commit_count  = $commit;
            @id_buffer     = ();
            @commit_buffer = ();
        }
    }

    # There are probably uncommitted records
    $indexer->update_index( \@id_buffer, \@commit_buffer );
    _log( 1, "$count records indexed.\n" );
}

# Checks some basic stuff to ensure that it's sane before we start.

    die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
}

# Output progress information.
#
#   _log($level, $msg);
#
# Will output $msg if the verbosity setting is set to $level or more. Will
# not include a trailing newline.
sub _log {
    my ($level, $msg) = @_;

    print $msg if ($verbose >= $level);
}
- 

Return to bug 18948

Lines 41-46 Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the i Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (-34 / +99 lines)
41		41
42	=head1 FUNCTIONS	42	=head1 FUNCTIONS
43		43
		44	=head2 Koha::SearchEngine::Elasticsearch::Indexer->({index => $index_name});
		45
		46	$index_name should be biblios or authorities
		47	call our SUPER class and then set ES parameters
		48
		49	=cut
		50
		51	sub new {
		52	my $class = shift @_;
		53	my $self = $class->SUPER::new(@_);
		54	my $params = $self->get_elasticsearch_params();
		55	$self->store(
		56	Catmandu::Store::ElasticSearch->new(
		57	%$params,
		58	index_settings => $self->get_elasticsearch_settings(),
		59	index_mappings => $self->get_elasticsearch_mappings(),
		60	)
		61	);
		62	return $self;
		63	}
		64
44	=head2 $indexer->update_index($biblionums, $records);	65	=head2 $indexer->update_index($biblionums, $records);
45		66
46	C<$biblionums> is an arrayref containing the biblionumbers for the records.	67	C<$biblionums> is an arrayref containing the biblionumbers for the records.
Lines 67-84 sub update_index { Link Here
67	}	88	}
68		89
69	my $from = $self->_convert_marc_to_json($records);	90	my $from = $self->_convert_marc_to_json($records);
70	if ( !$self->store ) {
71	my $params = $self->get_elasticsearch_params();
72	$self->store(
73	Catmandu::Store::ElasticSearch->new(
74	%$params,
75	index_settings => $self->get_elasticsearch_settings(),
76	index_mappings => $self->get_elasticsearch_mappings(),
77	)
78	);
79	}
80		91
81	#print Data::Dumper::Dumper( $from->to_array );	92	print Data::Dumper::Dumper( $from->to_array );
82	$self->store->bag->add_many($from);	93	$self->store->bag->add_many($from);
83	$self->store->bag->commit;	94	$self->store->bag->commit;
84	return 1;	95	return 1;
Lines 111-126 C<$biblionums> is an arrayref of biblionumbers to delete from the index. Link Here
111	sub delete_index {	122	sub delete_index {
112	my ($self, $biblionums) = @_;	123	my ($self, $biblionums) = @_;
113		124
114	if ( !$self->store ) {
115	my $params = $self->get_elasticsearch_params();
116	$self->store(
117	Catmandu::Store::ElasticSearch->new(
118	%$params,
119	index_settings => $self->get_elasticsearch_settings(),
120	index_mappings => $self->get_elasticsearch_mappings(),
121	)
122	);
123	}
124	$self->store->bag->delete($_) foreach @$biblionums;	125	$self->store->bag->delete($_) foreach @$biblionums;
125	$self->store->bag->commit;	126	$self->store->bag->commit;
126	}	127	}
Lines 149-168 after this will recreate it again. Link Here
149	sub drop_index {	150	sub drop_index {
150	my ($self) = @_;	151	my ($self) = @_;
151		152
152	if (!$self->store) {
153	# If this index doesn't exist, this will create it. Then it'll be
154	# deleted. That's not the end of the world however.
155	my $params = $self->get_elasticsearch_params();
156	$self->store(
157	Catmandu::Store::ElasticSearch->new(
158	%$params,
159	index_settings => $self->get_elasticsearch_settings(),
160	index_mappings => $self->get_elasticsearch_mappings(),
161	)
162	);
163	}
164	$self->store->drop();	153	$self->store->drop();
165	$self->store(undef);	154	my $params = $self->get_elasticsearch_params();
		155	$self->store(
		156	Catmandu::Store::ElasticSearch->new(
		157	%$params,
		158	index_settings => $self->get_elasticsearch_settings(),
		159	index_mappings => $self->get_elasticsearch_mappings(),
		160	)
		161	);
		162	}
		163
		164
		165	=head2 $indexer->do_reindex();
		166
		167	Performs a reindex based on records passed in
		168
		169	=cut
		170
		171	sub do_reindex {
		172	my ($self, $next, $delete, $commit, $verbose, $biblionumbers) = @_;
		173
		174	_log(1, $verbose, "Indexing ".$self->index."\n");
		175
		176	if ( scalar @$biblionumbers ) {
		177	if ( $self->index eq 'biblios' ) {
		178	$next = sub {
		179	my $r = shift @$biblionumbers;
		180	return () unless defined $r;
		181	return ( $r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
		182	};
		183	}
		184	elsif ( $self->index eq 'authorities' ) {
		185	$next = sub {
		186	my $r = shift @$biblionumbers;
		187	return () unless defined $r;
		188	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
		189	return ($r, $a->record);
		190	};
		191	}
		192	}
		193	else {
		194	my $records = $self->index eq 'biblios' ?
		195	Koha::BiblioUtils->get_all_biblios_iterator() : $self->index eq 'authorities' ?
		196	Koha::MetadataRecord::Authority->get_all_authorities_iterator() : die "Reindex called on undefined index";
		197	$next = sub { $records->next(); };
		198	}
		199
		200	warn
		201	$self->drop_index() if ( $delete );
		202
		203	my $count = 0;
		204	my $commit_count = $commit;
		205	my ( @id_buffer, @commit_buffer );
		206	while ( my $record = $next->() ) {
		207	my $id = $record->id;
		208	my $record = $record->record;
		209	_log( 1, $verbose, "$id\n" );
		210	$count++;
		211
		212	push @id_buffer, $id;
		213	push @commit_buffer, $record;
		214	if ( !( --$commit_count ) ) {
		215	_log( 2, $verbose, "Committing...\n" );
		216	$self->update_index( \@id_buffer, \@commit_buffer );
		217	$commit_count = $commit;
		218	@id_buffer = ();
219	@commit_buffer = ();
220	}
221	}
222	# There are probably uncommitted records (beacuse we didn't hit exact commit count)
223	$self->update_index( \@id_buffer, \@commit_buffer );
224	_log( 1, $verbose, "$count records indexed.\n" );
225	}
226
227
228	sub _log {
229	my ($level, $verbose, $msg) = @_;
230	print $msg if ($verbose >= $level);
166	}	231	}
167		232
168	sub _sanitise_records {	233	sub _sanitise_records {

Lines 122-195 pod2usage( -exitstatus => 0, -verbose => 2 ) if $man; Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-77 / +7 lines)
122	sanity_check();	122	sanity_check();
123		123
124	my $next;	124	my $next;
		125	print "$verbose verbosity";
125	if ($index_biblios) {	126	if ($index_biblios) {
126	_log(1, "Indexing biblios\n");	127	my $index_name = $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX;
127	if (@biblionumbers) {	128	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
128	$next = sub {	129	$indexer->do_reindex($next,$delete,$commit,$verbose,\@biblionumbers);
129	my $r = shift @biblionumbers;
130	return () unless defined $r;
131	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132	};
133	} else {
134	my $records = Koha::BiblioUtils->get_all_biblios_iterator();
135	$next = sub {
136	$records->next();
137	}
138	}
139	do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);
140	}	130	}
141	if ($index_authorities) {	131	if ($index_authorities) {
142	_log(1, "Indexing authorities\n");	132	my $index_name = $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX;
143	if (@biblionumbers) {	133	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new({ index => $index_name });
144	$next = sub {	134	$indexer->do_reindex($next,$delete,$commit,$verbose, \@biblionumbers);
145	my $r = shift @biblionumbers;
146	return () unless defined $r;
147	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
148	return ($r, $a->record);
149	};
150	} else {
151	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator();
152	$next = sub {
153	$records->next();
154	}
155	}
156	do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);
157	}
158
159	sub do_reindex {
160	my ( $next, $index_name ) = @_;
161
162	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
163	if ($delete) {
164
165	# We know it's safe to not recreate the indexer because update_index
166	# hasn't been called yet.
167	$indexer->drop_index();
168	}
169
170	my $count = 0;
171	my $commit_count = $commit;
172	my ( @id_buffer, @commit_buffer );
173	while ( my $record = $next->() ) {
174	my $id = $record->id;
175	my $record = $record->record;
176	_log( 1, "$id\n" );
177	$count++;
178
179	push @id_buffer, $id;
180	push @commit_buffer, $record;
181	if ( !( --$commit_count ) ) {
182	_log( 2, "Committing...\n" );
183	$indexer->update_index( \@id_buffer, \@commit_buffer );
184	$commit_count = $commit;
185	@id_buffer = ();
186	@commit_buffer = ();
187	}
188	}
189
190	# There are probably uncommitted records
191	$indexer->update_index( \@id_buffer, \@commit_buffer );
192	_log( 1, "$count records indexed.\n" );
193	}	135	}
194		136
195	# Checks some basic stuff to ensure that it's sane before we start.	137	# Checks some basic stuff to ensure that it's sane before we start.
Lines 199-212 sub sanity_check { Link Here
199	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );	141	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
200	}	142	}
201		143
202	# Output progress information.
203	#
204	# _log($level, $msg);
205	#
206	# Will output $msg if the verbosity setting is set to $level or more. Will
207	# not include a trailing newline.
208	sub _log {
209	my ($level, $msg) = @_;
210
211	print $msg if ($verbose >= $level);
212	}
213	-