From 517d1787f1515e00ada40da1a2a3e71e68af5860 Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Fri, 21 Jan 2022 14:30:15 +0100 Subject: [PATCH] Bug 29921: Allow to index in ES all records modified since a given date This patch adds an option `--modified-since` to rebuild_elasticsearch.pl If used, all records modified since the given date will be indexed. For biblios it uses biblio_metadata.timestamp and items.timestamp For authorities it uses auth_header.modification_time. Test plan: 1. Create several biblio records and items at different times (or update manually the columns specified above) 2. Look at the records modification time: SELECT MAX(GREATEST(bm.timestamp, COALESCE(i.timestamp, 0))) ts FROM biblio_metadata bm LEFT JOIN items i ON (bm.biblionumber = i.biblionumber) GROUP BY bm.biblionumber ORDER BY ts 3. Run misc/search_tools/rebuild_elasticsearch.pl -b -v --modified-since "$DATE" $DATE can be a date (YYYY-MM-DD) or a datetime (YYYY-MM-DD HH:MM[:SS]) 4. Verify that the correct number of records have been indexed (do the same for authorities:) 5. Create several authority records at different times (or update manually the columns specified above) 6. Look at the records modification time: SELECT modification_time FROM auth_header ORDER BY modification_time 7. Run misc/search_tools/rebuild_elasticsearch.pl -a -v --modified-since "$DATE" $DATE can be a date (YYYY-MM-DD) or a datetime (YYYY-MM-DD HH:MM[:SS]) 8. Verify that the correct number of records have been indexed --- Koha/BiblioUtils.pm | 20 ++++++++++++++++---- Koha/MetadataRecord/Authority.pm | 20 +++++++++----------- misc/search_tools/rebuild_elasticsearch.pl | 12 ++++++++++++ 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/Koha/BiblioUtils.pm b/Koha/BiblioUtils.pm index 1ece4a8317..2473baaaec 100644 --- a/Koha/BiblioUtils.pm +++ b/Koha/BiblioUtils.pm @@ -125,15 +125,27 @@ records instead of all. sub get_all_biblios_iterator { my ($self, %options) = @_; - my $search_terms = {}; + my @search_terms; + my $search_options = { + columns => [qw/ biblionumber /], + distinct => 1, + }; + my ($slice_modulo, $slice_count); if ($options{slice}) { $slice_count = $options{slice}->{count}; $slice_modulo = $options{slice}->{index}; - $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ]; + push @search_terms, \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ]; + } + + if ($options{modified_since}) { + $search_options->{join} = ['biblio_metadatas', 'items']; + push @search_terms, [ + { 'biblio_metadatas.timestamp' => { '>=', $options{modified_since} } }, + { 'items.timestamp' => { '>=', $options{modified_since} } }, + ]; } - my $search_options = { columns => [qw/ biblionumber /] }; if ( $options{desc} ){ $search_options->{order_by} = { -desc => 'biblionumber' }; } @@ -142,7 +154,7 @@ sub get_all_biblios_iterator { my $schema = $database->schema(); my $rs = $schema->resultset('Biblio')->search( - $search_terms, + [ -and => \@search_terms ], $search_options ); my $next_func = sub { # Warn and skip bad records, otherwise we break the loop diff --git a/Koha/MetadataRecord/Authority.pm b/Koha/MetadataRecord/Authority.pm index d16970a828..9c7c15fca5 100644 --- a/Koha/MetadataRecord/Authority.pm +++ b/Koha/MetadataRecord/Authority.pm @@ -175,22 +175,20 @@ records instead of all. sub get_all_authorities_iterator { my ($self, %options) = @_; - my $search_terms = { - marcxml => { '!=', undef } - }; + my @search_terms = ( { marcxml => { '!=', undef } } ); + my $search_options->{columns} = [qw/ authid /]; + my ($slice_modulo, $slice_count); if ($options{slice}) { $slice_count = $options{slice}->{count}; $slice_modulo = $options{slice}->{index}; - $search_terms = { - '-and' => [ - %{$search_terms}, - \[ 'mod(authid, ?) = ?', $slice_count, $slice_modulo ] - ] - }; + push @search_terms, \[ 'mod(authid, ?) = ?', $slice_count, $slice_modulo ]; + } + + if ($options{modified_since}) { + push @search_terms, { 'modification_time' => { '>=', $options{modified_since} } }; } - my $search_options->{columns} = [qw/ authid /]; if ($options{desc}) { $search_options->{order_by} = { -desc => 'authid' }; } @@ -199,7 +197,7 @@ sub get_all_authorities_iterator { my $schema = $database->schema(); my $rs = $schema->resultset('AuthHeader')->search( - $search_terms, + [ -and => \@search_terms ], $search_options); my $next_func = sub { # Warn and skip bad records, otherwise we break the loop diff --git a/misc/search_tools/rebuild_elasticsearch.pl b/misc/search_tools/rebuild_elasticsearch.pl index 337ac19dd2..98799776bb 100755 --- a/misc/search_tools/rebuild_elasticsearch.pl +++ b/misc/search_tools/rebuild_elasticsearch.pl @@ -34,6 +34,7 @@ B [B<--desc>] [B<-bn|--bnumber>] [B<-ai|--authid>] +[B<--modified-since=DATETIME>] [B<-p|--processes>] [B<-v|--verbose>] [B<-h|--help>] @@ -87,6 +88,11 @@ repeated. Only index the supplied authority id, mostly for testing purposes. May be repeated. +=item B<--modified-since=DATETIME> + +Only index records modified since supplied datetime. It cannot be used with +--bnumber or --authid options. + =item B<-p|--processes> Number of processes to use for indexing. This can be used to do more indexing @@ -128,6 +134,7 @@ my $verbose = 0; my $commit = 5000; my ($delete, $reset, $help, $man, $processes); my ($index_biblios, $index_authorities); +my $modified_since; my (@biblionumbers,@authids); my $desc; @@ -142,6 +149,7 @@ GetOptions( 'desc' => \$desc, 'bn|bnumber=i' => \@biblionumbers, 'ai|authid=i' => \@authids, + 'modified-since=s' => \$modified_since, 'p|processes=i' => \$processes, 'v|verbose+' => \$verbose, 'h|help' => \$help, @@ -193,6 +201,10 @@ if ($slice_count > 1) { $iterator_options{slice} = { index => $slice_index, count => $slice_count }; } +if ($modified_since) { + $iterator_options{modified_since} = $modified_since; +} + if( $desc ){ $iterator_options{desc} = 1; } -- 2.30.2