Attachment #110381 for bug #25265

View | Details | Raw Unified | Return to bug 25265
Collapse All | Expand All




use Koha::Authority;
use Koha::Libraries;
use Koha::SearchEngine;
use Koha::SearchEngine::Indexer;
use Koha::SearchEngine::Search;

use vars qw(@ISA @EXPORT);

    $record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
    # Update
    $dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::AUTHORITIES_INDEX });
    $indexer->index_records( $authid, "specialUpdate", "authorityserver", $record );

    return ( $authid );
}

    merge({ mergefrom => $authid }) if !$skip_merge;
    $dbh->do( "DELETE FROM auth_header WHERE authid=?", undef, $authid );
    logaction( "AUTHORITIES", "DELETE", $authid, "authority" ) if C4::Context->preference("AuthoritiesLog");
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::AUTHORITIES_INDEX });
    $indexer->index_records( $authid, "recordDelete", "authorityserver", undef );
}

=head2 ModAuthority




use Koha::ItemTypes;
use Koha::Plugins;
use Koha::SearchEngine;
use Koha::SearchEngine::Indexer;
use Koha::Libraries;
use Koha::Util::MARC;


=cut

sub DelBiblio {
    my ($biblionumber, $params) = @_;

    my $biblio = Koha::Biblios->find( $biblionumber );
    return unless $biblio; # Should we throw an exception instead?

        $hold->cancel;
    }

    unless ( $params->{skip_record_index} ){
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
        $indexer->index_records( $biblionumber, "recordDelete", "biblioserver" );
    }
    ModZebra( $biblionumber, "recordDelete", "biblioserver" );

    # delete biblioitems and items from Koha tables and save in deletedbiblioitems,deleteditems
    $sth = $dbh->prepare("SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");


$server is the server that we want to update

$record is the updated MARC record. If it's not supplied
and is needed it will be loaded from the database.

=cut

sub ModZebra {
###Accepts a $server variable thus we can use it for biblios authorities or other zebra dbs
    my ( $biblionumber, $op, $server ) = @_;
    $debug && warn "ModZebra: updates requested for: $biblionumber $op $server\n";
    if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) {

        # TODO abstract to a standard API that'll work for whatever
        require Koha::SearchEngine::Elasticsearch::Indexer;
        my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
            {
                index => $server eq 'biblioserver'
                ? $Koha::SearchEngine::BIBLIOS_INDEX
                : $Koha::SearchEngine::AUTHORITIES_INDEX
            }
        );
        if ( $op eq 'specialUpdate' ) {
            unless ($record) {
                $record = GetMarcBiblio({
                    biblionumber => $biblionumber,
                    embed_items  => 1 });
            }
            $indexer->update_index_background( [$biblionumber], [$record] );
        }
        elsif ( $op eq 'recordDelete' ) {
            $indexer->delete_index_background( [$biblionumber] );
        }
        else {
            croak "ModZebra called with unknown operation: $op";
        }
    }

    my $dbh = C4::Context->dbh;

    # true ModZebra commented until indexdata fixes zebraDB crashes (it seems they occur on multiple updates

    $m_rs->metadata( $record->as_xml_record($encoding) );
    $m_rs->store;

    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );

    return $biblionumber;
}




    MarkIssueReturned( $borrowernumber, $itemnum );
}

=head2 LostItem

  LostItem( $itemnumber, $mark_lost_from, $force_mark_returned, [$params] );

The final optional parameter, C<$params>, expected to contain
'skip_modzebra_update' key, which relayed down to Koha::Item/store,
there it prevents calling of ModZebra index_records,
which takes most of the time in batch adds/deletes: index_records better
to be called later in C<additem.pl> after the whole loop.

$params:
    skip_modzebra_update => 1|0

=cut

sub LostItem{
    my ($itemnumber, $mark_lost_from, $force_mark_returned, $params) = @_;

    unless ( $mark_lost_from ) {
        # Temporary check to avoid regressions


    #When item is marked lost automatically cancel its outstanding transfers and set items holdingbranch to the transfer source branch (frombranch)
    if (my ( $datesent,$frombranch,$tobranch ) = GetTransfers($itemnumber)) {
        Koha::Items->find($itemnumber)->holdingbranch($frombranch)->store({ skip_modzebra_update => $params->{skip_modzebra_update} });
    }
    my $transferdeleted = DeleteTransfer($itemnumber);
}




use Koha::Items;
use Koha::ItemTypes;
use Koha::SearchEngine;
use Koha::SearchEngine::Indexer;
use Koha::SearchEngine::Search;
use Koha::Libraries;



The final optional parameter, C<$params>, expected to contain
'skip_modzebra_update' key, which relayed down to Koha::Item/store,
there it prevents calling of index_records,
which takes most of the time in batch adds/deletes: index_records
to be called later in C<additem.pl> after the whole loop.

$params:

    return (\@itemnumbers, \@errors);
}

=head2 ModItemFromMarc

my $item = ModItemFromMarc($item_marc, $biblionumber, $itemnumber[, $params]);

The final optional parameter, C<$params>, expected to contain
'skip_modzebra_update' key, which relayed down to Koha::Item/store,
there it prevents calling of index_records,
which takes most of the time in batch adds/deletes: index_records better
to be called later in C<additem.pl> after the whole loop.

$params:
    skip_modzebra_update => 1|0

=cut

sub ModItemFromMarc {
    my ( $item_marc, $biblionumber, $itemnumber, $params ) = @_;
    my $biblionumber = shift;
    my $itemnumber = shift;

    my $frameworkcode = C4::Biblio::GetFrameworkCode($biblionumber);
    my ( $itemtag, $itemsubfield ) = C4::Biblio::GetMarcFromKohaField( "items.itemnumber" );

    $item_object = $item_object->set_or_blank($item);
    my $unlinked_item_subfields = _get_unlinked_item_subfields( $localitemmarc, $frameworkcode );
    $item_object->more_subfields_xml(_get_unlinked_subfields_xml($unlinked_item_subfields));
    $item_object->store({ skip_modzebra_update => $params->{skip_modzebra_update} });

    return $item_object->unblessed;
}

            AND biblionumber = ?
    |, undef, $tobiblioitem, $tobiblio, $itemnumber, $frombiblio );
    if ($return == 1) {
        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
        $indexer->index_records( $tobiblio, "specialUpdate", "biblioserver" );
        $indexer->index_records( $frombiblio, "specialUpdate", "biblioserver" );
	    # Checking if the item we want to move is in an order 
        require C4::Acquisition;
        my $order = C4::Acquisition::GetOrderFromItemnumber($itemnumber);




use C4::Context;
use C4::Circulation;
use C4::Reserves;
use C4::Biblio qw( ModZebra ); # FIXME This is terrible, we should move the indexation code outside of C4::Biblio
use C4::ClassSource; # FIXME We would like to avoid that
use C4::Log qw( logaction );

use Koha::Checkouts;
use Koha::CirculationRules;
use Koha::SearchEngine::Indexer;
use Koha::Item::Transfer::Limits;
use Koha::Item::Transfers;
use Koha::ItemTypes;

    $item->store;

$params can take an optional 'skip_modzebra_update' parameter.
If set, the reindexation process will not happen (index_records not called)

NOTE: This is a temporary fix to answer a performance issue when lot of items
are added (or modified) at the same time.

            $self->cn_sort($cn_sort);
        }

        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
        $indexer->index_records( $self->biblionumber, "specialUpdate", "biblioserver" )
            unless $params->{skip_modzebra_update};

        logaction( "CATALOGUING", "ADD", $self->itemnumber, "item" )

            $self->paidfor('');
        }

        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
        $indexer->index_records( $self->biblionumber, "specialUpdate", "biblioserver" )
            unless $params->{skip_modzebra_update};

        logaction( "CATALOGUING", "MODIFY", $self->itemnumber, "item " . Dumper($self->unblessed) )

    # FIXME check the item has no current issues
    # i.e. raise the appropriate exception

    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
    $indexer->index_records( $self->biblionumber, "specialUpdate", "biblioserver" )
        unless $params->{skip_modzebra_update};

    $self->_after_item_action_hooks({ action => 'delete' });

Lines 25-30 use base qw(Koha::SearchEngine::Elasticsearch); Link Here

(-)a/Koha/SearchEngine/Elasticsearch/Indexer.pm (+37 lines)
25	use Data::Dumper;	25	use Data::Dumper;
26		26
27	use Koha::Exceptions;	27	use Koha::Exceptions;
		28	use Koha::SearchEngine::Zebra::Indexer;
		29	use C4::Biblio;
28	use C4::Context;	30	use C4::Context;
29		31
30	=head1 NAME	32	=head1 NAME
Lines 280-285 sub update_index_background { Link Here
280	$self->update_index(@_);	282	$self->update_index(@_);
281	}	283	}
282		284
		285	=head2 index_records
		286
		287	This function takes an array of biblionumbers and fetches the records to send to update_index
		288	for actual indexing.
		289
		290	If $records parameter is provided the records will be used as-is, this is only utilized for authorities
		291	at the moment.
		292
		293	The other variables are used for parity with Zebra indexing calls. Currently the calls are passed through
		294	to Zebra as well.
		295
		296	=cut
		297
		298	sub index_records {
		299	my ( $self, $biblionumbers, $op, $server, $records ) = @_;
		300	$biblionumbers = [$biblionumbers] if ref $biblionumbers ne 'ARRAY' && defined $biblionumbers;
		301	$records = [$records] if ref $biblionumbers ne 'ARRAY' && defined $records;
		302	if ( $op eq 'specialUpdate' ) {
		303	unless ($records) {
		304	foreach my $biblionumber ( @$biblionumbers ){
		305	my $record = C4::Biblio::GetMarcBiblio({
		306	biblionumber => $biblionumber,
		307	embed_items => 1 });
		308	push @$records, $record;
		309	}
		310	}
		311	$self->update_index_background( $biblionumbers, $records );
		312	}
		313	elsif ( $op eq 'recordDelete' ) {
		314	$self->delete_index_background( $biblionumbers );
		315	}
		316	#FIXME Current behaviour is to index Zebra when using ES, at some point we should stop
		317	Koha::SearchEngine::Zebra::Indexer::index_records( $self, $biblionumbers, $op, $server, undef );
		318	}
		319
283	=head2 delete_index($biblionums)	320	=head2 delete_index($biblionums)
284		321
285	C<$biblionums> is an arrayref of biblionumbers to delete from the index.	322	C<$biblionums> is an arrayref of biblionumbers to delete from the index.




package Koha::SearchEngine::Indexer;

# This file is part of Koha.
#
# Copyright 2020 ByWater Solutions
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

# This is a shim that gives you the appropriate search object for your
# system preference.

=head1 NAME

Koha::SearchEngine::Indexer - instantiate the search object that corresponds to
the C<SearchEngine> system preference.

=head1 DESCRIPTION

This allows you to be agnostic about what the search engine configuration is
and just get whatever indexer object you need.

=head1 SYNOPSIS

    use Koha::SearchEngine::Indexer;
    my $searcher = Koha::SearchEngine::Indexer->new({ index => Koha::SearchEngine::Index});

=head1 METHODS

=head2 new

Creates a new C<Search> of whatever the relevant type is.

=cut

use Modern::Perl;
use C4::Context;
use C4::Biblio qw//;

sub new {
    my $engine = C4::Context->preference("SearchEngine") // 'Zebra';
    my $file = "Koha/SearchEngine/${engine}/Indexer.pm";
    my $class = "Koha::SearchEngine::${engine}::Indexer";
    require $file;
    shift @_;
    return $class->new(@_);
}

1;




package Koha::SearchEngine::Zebra::Indexer;

# Copyright 2020 ByWater Solutions
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.

use Modern::Perl;
use C4::Biblio qw(ModZebra); # FIXME This is terrible, we should move the indexation code outside of C4::Biblio
use base qw(Class::Accessor);

=head1 NAME

Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the index

=head1 SYNOPSIS

    my $indexer = Koha::SearchEngine::Zebra::Indexer->new();
    $indexer->index_records( $biblionumbers, $op, $server, $records);


=head1 FUNCTIONS

=head2 new

    This is a dummy function to create the object. C4::Biblio->ModZebra is doing the real work
    now and needed variables are passed to index_records

=cut

sub new {
    my $class = shift @_;
    my $self = $class->SUPER::new(@_);
}

=head2 index_records($biblionumbers, $op, $server, $records)

    This is simply a wrapper to C4::Biblio::ModZebra that takes an array of records and
    passes them through individually

    The final parameter $records is not used in Zebra, it exists for parity with Elasticsearch calls

=cut

sub index_records {
    my ( $self, $biblionumbers, $op, $server, $records ) = @_;
    $biblionumbers = [$biblionumbers] if ref $biblionumbers ne 'ARRAY' && defined $biblionumbers;
    foreach my $biblionumber ( @$biblionumbers ){
        ModZebra( $biblionumber, $op, $server );
    }
}

1;




use Koha::ItemTypes;
use Koha::Libraries;
use Koha::Patrons;
use Koha::SearchEngine::Indexer;
use List::MoreUtils qw/any/;
use C4::Search;
use Storable qw(thaw freeze);

		$oldbarcode = $barcodevalue;
	    }

        my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
        $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );

	    undef($itemrecord);
	}

        next if ref $error eq 'Koha::Item'; # Deleted item is returned if deletion successful
        push @errors,$error;
    }
    my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
    $indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
    if ( @errors ) {
        $nextop="additem";
    } else {

Lines 44-49 use Koha::DateUtils; Link Here

(-)a/tools/batchMod.pl (-6 / +25 lines)
44	use Koha::Items;	44	use Koha::Items;
45	use Koha::ItemTypes;	45	use Koha::ItemTypes;
46	use Koha::Patrons;	46	use Koha::Patrons;
		47	use Koha::SearchEngine::Indexer;
47		48
48	my $input = new CGI;	49	my $input = new CGI;
49	my $dbh = C4::Context->dbh;	50	my $dbh = C4::Context->dbh;
Lines 192-197 if ($op eq "action") { Link Here
192	$ynhash->{'av'.$yn->authorised_value} = $yn->lib;	193	$ynhash->{'av'.$yn->authorised_value} = $yn->lib;
193	}	194	}
194		195
		196	my $upd_biblionumbers;
		197	my $del_biblionumbers;
195	try {	198	try {
196	my $schema = Koha::Database->new->schema;	199	my $schema = Koha::Database->new->schema;
197	$schema->txn_do(	200	$schema->txn_do(
Lines 210-215 if ($op eq "action") { Link Here
210	my $return = $item->safe_delete;	213	my $return = $item->safe_delete;
211	if ( ref( $return ) ) {	214	if ( ref( $return ) ) {
212	$deleted_items++;	215	$deleted_items++;
		216	push @$upd_biblionumbers, $itemdata->{'biblionumber'};
213	}	217	}
214	else {	218	else {
215	$not_deleted_items++;	219	$not_deleted_items++;
Lines 227-235 if ($op eq "action") { Link Here
227	if ($del_records) {	231	if ($del_records) {
228	my $itemscount = Koha::Biblios->find( $itemdata->{'biblionumber'} )->items->count;	232	my $itemscount = Koha::Biblios->find( $itemdata->{'biblionumber'} )->items->count;
229	if ( $itemscount == 0 ) {	233	if ( $itemscount == 0 ) {
230	my $error = DelBiblio( $itemdata->{'biblionumber'} );	234	my $error = DelBiblio( $itemdata->{'biblionumber'}, { skip_record_index => 1 } );
231	unless ($error) {	235	unless ($error) {
232	$deleted_records++;	236	$deleted_records++;
		237	push @$del_biblionumbers, $itemdata->{'biblionumber'};
233	if ( $src eq 'CATALOGUING' ) {	238	if ( $src eq 'CATALOGUING' ) {
234	# We are coming catalogue/detail.pl, there were items from a single bib record	239	# We are coming catalogue/detail.pl, there were items from a single bib record
235	$template->param( biblio_deleted => 1 );	240	$template->param( biblio_deleted => 1 );
Lines 298-312 if ($op eq "action") { Link Here
298	my $item = ModItemFromMarc(	303	my $item = ModItemFromMarc(
299	$localmarcitem,	304	$localmarcitem,
300	$itemdata->{biblionumber},	305	$itemdata->{biblionumber},
301	$itemnumber	306	$itemnumber,
		307	{ skip_modzebra_update => 1 },
302	)	308	)
303	)	309	)
304	{	310	{
305	LostItem( $itemnumber, 'batchmod' )	311	LostItem(
306	if $item->{itemlost}	312	$itemnumber,
		313	'batchmod',
		314	undef,
		315	{ skip_modzebra_update => 1 }
		316	) if $item->{itemlost}
307	and not $itemdata->{itemlost};	317	and not $itemdata->{itemlost};
308	}	318	}
309	};	319	};
		320	push @$upd_biblionumbers, $itemdata->{'biblionumber'};
310	}	321	}
311	}	322	}
312	if ($runinbackground) {	323	if ($runinbackground) {
Lines 336-342 if ($op eq "action") { Link Here
336	}	347	}
337	die "Something terrible has happened!"	348	die "Something terrible has happened!"
338	if ($_ =~ /Rollback failed/); # Rollback failed	349	if ($_ =~ /Rollback failed/); # Rollback failed
339	}	350	};
		351	$upd_biblionumbers = [ uniq @$upd_biblionumbers ]; # Only update each bib once
		352
		353	# Don't send specialUpdate for records we are going to delete
		354	my %del_bib_hash = map{ $_ => undef } @$del_biblionumbers;
		355	@$upd_biblionumbers = grep( ! exists( $del_bib_hash{$_} ), @$upd_biblionumbers );
		356
		357	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		358	$indexer->index_records( $upd_biblionumbers, 'specialUpdate', "biblioserver", undef ) if @$upd_biblionumbers;
		359	$indexer->index_records( $del_biblionumbers, 'recordDelete', "biblioserver", undef ) if @$del_biblionumbers;
340	}	360	}
341	}	361	}
342	#	362	#
343	-

Return to bug 25265

Lines 103-108 use Koha::Holds; Link Here

(-)a/C4/Biblio.pm (-39 / +10 lines)
103	use Koha::ItemTypes;	103	use Koha::ItemTypes;
104	use Koha::Plugins;	104	use Koha::Plugins;
105	use Koha::SearchEngine;	105	use Koha::SearchEngine;
		106	use Koha::SearchEngine::Indexer;
106	use Koha::Libraries;	107	use Koha::Libraries;
107	use Koha::Util::MARC;	108	use Koha::Util::MARC;
108		109
Lines 366-372 C<$error> : undef unless an error occurs Link Here
366	=cut	367	=cut
367		368
368	sub DelBiblio {	369	sub DelBiblio {
369	my ($biblionumber) = @_;	370	my ($biblionumber, $params) = @_;
370		371
371	my $biblio = Koha::Biblios->find( $biblionumber );	372	my $biblio = Koha::Biblios->find( $biblionumber );
372	return unless $biblio; # Should we throw an exception instead?	373	return unless $biblio; # Should we throw an exception instead?
Lines 391-401 sub DelBiblio { Link Here
391	$hold->cancel;	392	$hold->cancel;
392	}	393	}
393		394
394	# Delete in Zebra. Be careful NOT to move this line after _koha_delete_biblio	395	unless ( $params->{skip_record_index} ){
395	# for at least 2 reasons :	396	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
396	# - if something goes wrong, the biblio may be deleted from Koha but not from zebra	397	$indexer->index_records( $biblionumber, "recordDelete", "biblioserver" );
397	# and we would have no way to remove it (except manually in zebra, but I bet it would be very hard to handle the problem)	398	}
398	ModZebra( $biblionumber, "recordDelete", "biblioserver" );
399		399
400	# delete biblioitems and items from Koha tables and save in deletedbiblioitems,deleteditems	400	# delete biblioitems and items from Koha tables and save in deletedbiblioitems,deleteditems
401	$sth = $dbh->prepare("SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");	401	$sth = $dbh->prepare("SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
Lines 2497-2538 $op is specialUpdate or recordDelete, and is used to know what we want to do Link Here
2497		2497
2498	$server is the server that we want to update	2498	$server is the server that we want to update
2499		2499
2500	$record is the updated MARC record. If it's not supplied
2501	and is needed it will be loaded from the database.
2502
2503	=cut	2500	=cut
2504		2501
2505	sub ModZebra {	2502	sub ModZebra {
2506	###Accepts a $server variable thus we can use it for biblios authorities or other zebra dbs	2503	###Accepts a $server variable thus we can use it for biblios authorities or other zebra dbs
2507	my ( $biblionumber, $op, $server, $record ) = @_;	2504	my ( $biblionumber, $op, $server ) = @_;
2508	$debug && warn "ModZebra: update requested for: $biblionumber $op $server\n";	2505	$debug && warn "ModZebra: updates requested for: $biblionumber $op $server\n";
2509	if ( C4::Context->preference('SearchEngine') eq 'Elasticsearch' ) {
2510
2511	# TODO abstract to a standard API that'll work for whatever
2512	require Koha::SearchEngine::Elasticsearch::Indexer;
2513	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new(
2514	{
2515	index => $server eq 'biblioserver'
2516	? $Koha::SearchEngine::BIBLIOS_INDEX
2517	: $Koha::SearchEngine::AUTHORITIES_INDEX
2518	}
2519	);
2520	if ( $op eq 'specialUpdate' ) {
2521	unless ($record) {
2522	$record = GetMarcBiblio({
2523	biblionumber => $biblionumber,
2524	embed_items => 1 });
2525	}
2526	$indexer->update_index_background( [$biblionumber], [$record] );
2527	}
2528	elsif ( $op eq 'recordDelete' ) {
2529	$indexer->delete_index_background( [$biblionumber] );
2530	}
2531	else {
2532	croak "ModZebra called with unknown operation: $op";
2533	}
2534	}
2535
2536	my $dbh = C4::Context->dbh;	2506	my $dbh = C4::Context->dbh;
2537		2507
2538	# true ModZebra commented until indexdata fixes zebraDB crashes (it seems they occur on multiple updates	2508	# true ModZebra commented until indexdata fixes zebraDB crashes (it seems they occur on multiple updates
Lines 3147-3153 sub ModBiblioMarc { Link Here
3147	$m_rs->metadata( $record->as_xml_record($encoding) );	3117	$m_rs->metadata( $record->as_xml_record($encoding) );
3148	$m_rs->store;	3118	$m_rs->store;
3149		3119
3150	ModZebra( $biblionumber, "specialUpdate", "biblioserver" );	3120	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		3121	$indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
3151		3122
3152	return $biblionumber;	3123	return $biblionumber;
3153	}	3124	}

Lines 3716-3724 sub ReturnLostItem{ Link Here

(-)a/C4/Circulation.pm (-2 / +16 lines)
3716	MarkIssueReturned( $borrowernumber, $itemnum );	3716	MarkIssueReturned( $borrowernumber, $itemnum );
3717	}	3717	}
3718		3718
		3719	=head2 LostItem
		3720
		3721	LostItem( $itemnumber, $mark_lost_from, $force_mark_returned, [$params] );
		3722
		3723	The final optional parameter, C<$params>, expected to contain
		3724	'skip_modzebra_update' key, which relayed down to Koha::Item/store,
		3725	there it prevents calling of ModZebra index_records,
		3726	which takes most of the time in batch adds/deletes: index_records better
		3727	to be called later in C<additem.pl> after the whole loop.
		3728
		3729	$params:
		3730	skip_modzebra_update => 1\|0
		3731
		3732	=cut
3719		3733
3720	sub LostItem{	3734	sub LostItem{
3721	my ($itemnumber, $mark_lost_from, $force_mark_returned) = @_;	3735	my ($itemnumber, $mark_lost_from, $force_mark_returned, $params) = @_;
3722		3736
3723	unless ( $mark_lost_from ) {	3737	unless ( $mark_lost_from ) {
3724	# Temporary check to avoid regressions	3738	# Temporary check to avoid regressions
Lines 3769-3775 sub LostItem{ Link Here
3769		3783
3770	#When item is marked lost automatically cancel its outstanding transfers and set items holdingbranch to the transfer source branch (frombranch)	3784	#When item is marked lost automatically cancel its outstanding transfers and set items holdingbranch to the transfer source branch (frombranch)
3771	if (my ( $datesent,$frombranch,$tobranch ) = GetTransfers($itemnumber)) {	3785	if (my ( $datesent,$frombranch,$tobranch ) = GetTransfers($itemnumber)) {
3772	Koha::Items->find($itemnumber)->holdingbranch($frombranch)->store;	3786	Koha::Items->find($itemnumber)->holdingbranch($frombranch)->store({ skip_modzebra_update => $params->{skip_modzebra_update} });
3773	}	3787	}
3774	my $transferdeleted = DeleteTransfer($itemnumber);	3788	my $transferdeleted = DeleteTransfer($itemnumber);
3775	}	3789	}

Lines 34-39 use Koha::Authority::Types; Link Here

(-)a/C4/AuthoritiesMarc.pm (-2 / +5 lines)
34	use Koha::Authority;	34	use Koha::Authority;
35	use Koha::Libraries;	35	use Koha::Libraries;
36	use Koha::SearchEngine;	36	use Koha::SearchEngine;
		37	use Koha::SearchEngine::Indexer;
37	use Koha::SearchEngine::Search;	38	use Koha::SearchEngine::Search;
38		39
39	use vars qw(@ISA @EXPORT);	40	use vars qw(@ISA @EXPORT);
Lines 628-634 sub AddAuthority { Link Here
628	$record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );	629	$record->insert_fields_ordered( MARC::Field->new( '001', $authid ) );
629	# Update	630	# Update
630	$dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;	631	$dbh->do( "UPDATE auth_header SET authtypecode=?, marc=?, marcxml=? WHERE authid=?", undef, $authtypecode, $record->as_usmarc, $record->as_xml_record($format), $authid ) or die $DBI::errstr;
631	ModZebra( $authid, 'specialUpdate', 'authorityserver', $record );	632	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::AUTHORITIES_INDEX });
		633	$indexer->index_records( $authid, "specialUpdate", "authorityserver", $record );
632		634
633	return ( $authid );	635	return ( $authid );
634	}	636	}
Lines 656-662 sub DelAuthority { Link Here
656	merge({ mergefrom => $authid }) if !$skip_merge;	658	merge({ mergefrom => $authid }) if !$skip_merge;
657	$dbh->do( "DELETE FROM auth_header WHERE authid=?", undef, $authid );	659	$dbh->do( "DELETE FROM auth_header WHERE authid=?", undef, $authid );
658	logaction( "AUTHORITIES", "DELETE", $authid, "authority" ) if C4::Context->preference("AuthoritiesLog");	660	logaction( "AUTHORITIES", "DELETE", $authid, "authority" ) if C4::Context->preference("AuthoritiesLog");
659	ModZebra( $authid, "recordDelete", "authorityserver", undef);	661	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::AUTHORITIES_INDEX });
		662	$indexer->index_records( $authid, "recordDelete", "authorityserver", undef );
660	}	663	}
661		664
662	=head2 ModAuthority	665	=head2 ModAuthority

Lines 70-75 use Koha::Biblioitems; Link Here

(-)a/C4/Items.pm (-8 / +23 lines)
70	use Koha::Items;	70	use Koha::Items;
71	use Koha::ItemTypes;	71	use Koha::ItemTypes;
72	use Koha::SearchEngine;	72	use Koha::SearchEngine;
		73	use Koha::SearchEngine::Indexer;
73	use Koha::SearchEngine::Search;	74	use Koha::SearchEngine::Search;
74	use Koha::Libraries;	75	use Koha::Libraries;
75		76
Lines 144-151 record and a biblionumber, create a new item record. Link Here
144		145
145	The final optional parameter, C<$params>, expected to contain	146	The final optional parameter, C<$params>, expected to contain
146	'skip_modzebra_update' key, which relayed down to Koha::Item/store,	147	'skip_modzebra_update' key, which relayed down to Koha::Item/store,
147	there it prevents calling of ModZebra (and Elasticsearch update),	148	there it prevents calling of index_records,
148	which takes most of the time in batch adds/deletes: ModZebra better	149	which takes most of the time in batch adds/deletes: index_records
149	to be called later in C<additem.pl> after the whole loop.	150	to be called later in C<additem.pl> after the whole loop.
150		151
151	$params:	152	$params:
Lines 281-290 sub AddItemBatchFromMarc { Link Here
281	return (\@itemnumbers, \@errors);	282	return (\@itemnumbers, \@errors);
282	}	283	}
283		284
		285	=head2 ModItemFromMarc
		286
		287	my $item = ModItemFromMarc($item_marc, $biblionumber, $itemnumber[, $params]);
		288
		289	The final optional parameter, C<$params>, expected to contain
		290	'skip_modzebra_update' key, which relayed down to Koha::Item/store,
		291	there it prevents calling of index_records,
		292	which takes most of the time in batch adds/deletes: index_records better
		293	to be called later in C<additem.pl> after the whole loop.
		294
		295	$params:
		296	skip_modzebra_update => 1\|0
		297
		298	=cut
		299
284	sub ModItemFromMarc {	300	sub ModItemFromMarc {
285	my $item_marc = shift;	301	my ( $item_marc, $biblionumber, $itemnumber, $params ) = @_;
286	my $biblionumber = shift;
287	my $itemnumber = shift;
288		302
289	my $frameworkcode = C4::Biblio::GetFrameworkCode($biblionumber);	303	my $frameworkcode = C4::Biblio::GetFrameworkCode($biblionumber);
290	my ( $itemtag, $itemsubfield ) = C4::Biblio::GetMarcFromKohaField( "items.itemnumber" );	304	my ( $itemtag, $itemsubfield ) = C4::Biblio::GetMarcFromKohaField( "items.itemnumber" );
Lines 313-319 sub ModItemFromMarc { Link Here
313	$item_object = $item_object->set_or_blank($item);	327	$item_object = $item_object->set_or_blank($item);
314	my $unlinked_item_subfields = _get_unlinked_item_subfields( $localitemmarc, $frameworkcode );	328	my $unlinked_item_subfields = _get_unlinked_item_subfields( $localitemmarc, $frameworkcode );
315	$item_object->more_subfields_xml(_get_unlinked_subfields_xml($unlinked_item_subfields));	329	$item_object->more_subfields_xml(_get_unlinked_subfields_xml($unlinked_item_subfields));
316	$item_object->store;	330	$item_object->store({ skip_modzebra_update => $params->{skip_modzebra_update} });
317		331
318	return $item_object->unblessed;	332	return $item_object->unblessed;
319	}	333	}
Lines 1094-1101 sub MoveItemFromBiblio { Link Here
1094	AND biblionumber = ?	1108	AND biblionumber = ?
1095	\|, undef, $tobiblioitem, $tobiblio, $itemnumber, $frombiblio );	1109	\|, undef, $tobiblioitem, $tobiblio, $itemnumber, $frombiblio );
1096	if ($return == 1) {	1110	if ($return == 1) {
1097	ModZebra( $tobiblio, "specialUpdate", "biblioserver" );	1111	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
1098	ModZebra( $frombiblio, "specialUpdate", "biblioserver" );	1112	$indexer->index_records( $tobiblio, "specialUpdate", "biblioserver" );
		1113	$indexer->index_records( $frombiblio, "specialUpdate", "biblioserver" );
1099	# Checking if the item we want to move is in an order	1114	# Checking if the item we want to move is in an order
1100	require C4::Acquisition;	1115	require C4::Acquisition;
1101	my $order = C4::Acquisition::GetOrderFromItemnumber($itemnumber);	1116	my $order = C4::Acquisition::GetOrderFromItemnumber($itemnumber);

Lines 30-41 use Koha::DateUtils qw( dt_from_string ); Link Here

(-)a/Koha/Item.pm (-5 / +8 lines)
30	use C4::Context;	30	use C4::Context;
31	use C4::Circulation;	31	use C4::Circulation;
32	use C4::Reserves;	32	use C4::Reserves;
33	use C4::Biblio qw( ModZebra ); # FIXME This is terrible, we should move the indexation code outside of C4::Biblio
34	use C4::ClassSource; # FIXME We would like to avoid that	33	use C4::ClassSource; # FIXME We would like to avoid that
35	use C4::Log qw( logaction );	34	use C4::Log qw( logaction );
36		35
37	use Koha::Checkouts;	36	use Koha::Checkouts;
38	use Koha::CirculationRules;	37	use Koha::CirculationRules;
		38	use Koha::SearchEngine::Indexer;
39	use Koha::Item::Transfer::Limits;	39	use Koha::Item::Transfer::Limits;
40	use Koha::Item::Transfers;	40	use Koha::Item::Transfers;
41	use Koha::ItemTypes;	41	use Koha::ItemTypes;
Lines 62-68 Koha::Item - Koha Item object class Link Here
62	$item->store;	62	$item->store;
63		63
64	$params can take an optional 'skip_modzebra_update' parameter.	64	$params can take an optional 'skip_modzebra_update' parameter.
65	If set, the reindexation process will not happen (ModZebra not called)	65	If set, the reindexation process will not happen (index_records not called)
66		66
67	NOTE: This is a temporary fix to answer a performance issue when lot of items	67	NOTE: This is a temporary fix to answer a performance issue when lot of items
68	are added (or modified) at the same time.	68	are added (or modified) at the same time.
Lines 113-119 sub store { Link Here
113	$self->cn_sort($cn_sort);	113	$self->cn_sort($cn_sort);
114	}	114	}
115		115
116	C4::Biblio::ModZebra( $self->biblionumber, "specialUpdate", "biblioserver" )	116	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		117	$indexer->index_records( $self->biblionumber, "specialUpdate", "biblioserver" )
117	unless $params->{skip_modzebra_update};	118	unless $params->{skip_modzebra_update};
118		119
119	logaction( "CATALOGUING", "ADD", $self->itemnumber, "item" )	120	logaction( "CATALOGUING", "ADD", $self->itemnumber, "item" )
Lines 189-195 sub store { Link Here
189	$self->paidfor('');	190	$self->paidfor('');
190	}	191	}
191		192
192	C4::Biblio::ModZebra( $self->biblionumber, "specialUpdate", "biblioserver" )	193	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		194	$indexer->index_records( $self->biblionumber, "specialUpdate", "biblioserver" )
193	unless $params->{skip_modzebra_update};	195	unless $params->{skip_modzebra_update};
194		196
195	logaction( "CATALOGUING", "MODIFY", $self->itemnumber, "item " . Dumper($self->unblessed) )	197	logaction( "CATALOGUING", "MODIFY", $self->itemnumber, "item " . Dumper($self->unblessed) )
Lines 217-223 sub delete { Link Here
217	# FIXME check the item has no current issues	219	# FIXME check the item has no current issues
218	# i.e. raise the appropriate exception	220	# i.e. raise the appropriate exception
219		221
220	C4::Biblio::ModZebra( $self->biblionumber, "specialUpdate", "biblioserver" )	222	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		223	$indexer->index_records( $self->biblionumber, "specialUpdate", "biblioserver" )
221	unless $params->{skip_modzebra_update};	224	unless $params->{skip_modzebra_update};
222		225
223	$self->_after_item_action_hooks({ action => 'delete' });	226	$self->_after_item_action_hooks({ action => 'delete' });

Line 0 Link Here

(-)a/Koha/SearchEngine/Indexer.pm (+59 lines)
	1	package Koha::SearchEngine::Indexer;
	2
	3	# This file is part of Koha.
	4	#
	5	# Copyright 2020 ByWater Solutions
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it
	8	# under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 3 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but
	13	# WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	19
	20	# This is a shim that gives you the appropriate search object for your
	21	# system preference.
	22
	23	=head1 NAME
	24
	25	Koha::SearchEngine::Indexer - instantiate the search object that corresponds to
	26	the C<SearchEngine> system preference.
	27
	28	=head1 DESCRIPTION
	29
	30	This allows you to be agnostic about what the search engine configuration is
	31	and just get whatever indexer object you need.
	32
	33	=head1 SYNOPSIS
	34
	35	use Koha::SearchEngine::Indexer;
	36	my $searcher = Koha::SearchEngine::Indexer->new({ index => Koha::SearchEngine::Index});
	37
	38	=head1 METHODS
	39
	40	=head2 new
	41
	42	Creates a new C<Search> of whatever the relevant type is.
	43
	44	=cut
	45
	46	use Modern::Perl;
	47	use C4::Context;
	48	use C4::Biblio qw//;
	49
	50	sub new {
	51	my $engine = C4::Context->preference("SearchEngine") // 'Zebra';
	52	my $file = "Koha/SearchEngine/${engine}/Indexer.pm";
	53	my $class = "Koha::SearchEngine::${engine}::Indexer";
	54	require $file;
	55	shift @_;
	56	return $class->new(@_);
	57	}
	58
	59	1;

Line 0 Link Here

(-)a/Koha/SearchEngine/Zebra/Indexer.pm (+65 lines)
		1	package Koha::SearchEngine::Zebra::Indexer;
		2
		3	# Copyright 2020 ByWater Solutions
		4	#
		5	# This file is part of Koha.
		6	#
		7	# Koha is free software; you can redistribute it and/or modify it
		8	# under the terms of the GNU General Public License as published by
		9	# the Free Software Foundation; either version 3 of the License, or
		10	# (at your option) any later version.
		11	#
		12	# Koha is distributed in the hope that it will be useful, but
		13	# WITHOUT ANY WARRANTY; without even the implied warranty of
		14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		15	# GNU General Public License for more details.
		16	#
		17	# You should have received a copy of the GNU General Public License
		18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
		19
		20	use Modern::Perl;
		21	use C4::Biblio qw(ModZebra); # FIXME This is terrible, we should move the indexation code outside of C4::Biblio
		22	use base qw(Class::Accessor);
		23
		24	=head1 NAME
		25
		26	Koha::SearchEngine::Elasticsearch::Indexer - handles adding new records to the index
		27
		28	=head1 SYNOPSIS
		29
		30	my $indexer = Koha::SearchEngine::Zebra::Indexer->new();
		31	$indexer->index_records( $biblionumbers, $op, $server, $records);
		32
		33
		34	=head1 FUNCTIONS
		35
		36	=head2 new
		37
		38	This is a dummy function to create the object. C4::Biblio->ModZebra is doing the real work
		39	now and needed variables are passed to index_records
		40
		41	=cut
		42
		43	sub new {
		44	my $class = shift @_;
		45	my $self = $class->SUPER::new(@_);
		46	}
		47
		48	=head2 index_records($biblionumbers, $op, $server, $records)
		49
		50	This is simply a wrapper to C4::Biblio::ModZebra that takes an array of records and
		51	passes them through individually
		52
		53	The final parameter $records is not used in Zebra, it exists for parity with Elasticsearch calls
		54
		55	=cut
		56
		57	sub index_records {
		58	my ( $self, $biblionumbers, $op, $server, $records ) = @_;
		59	$biblionumbers = [$biblionumbers] if ref $biblionumbers ne 'ARRAY' && defined $biblionumbers;
		60	foreach my $biblionumber ( @$biblionumbers ){
		61	ModZebra( $biblionumber, $op, $server );
		62	}
		63	}
		64
65	1;

Lines 35-40 use Koha::Items; Link Here

(-)a/cataloguing/additem.pl (-2 / +5 lines)
35	use Koha::ItemTypes;	35	use Koha::ItemTypes;
36	use Koha::Libraries;	36	use Koha::Libraries;
37	use Koha::Patrons;	37	use Koha::Patrons;
		38	use Koha::SearchEngine::Indexer;
38	use List::MoreUtils qw/any/;	39	use List::MoreUtils qw/any/;
39	use C4::Search;	40	use C4::Search;
40	use Storable qw(thaw freeze);	41	use Storable qw(thaw freeze);
Lines 629-635 if ($op eq "additem") { Link Here
629	$oldbarcode = $barcodevalue;	630	$oldbarcode = $barcodevalue;
630	}	631	}
631		632
632	C4::Biblio::ModZebra( $biblionumber, "specialUpdate", "biblioserver" );	633	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		634	$indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
633		635
634	undef($itemrecord);	636	undef($itemrecord);
635	}	637	}
Lines 707-713 if ($op eq "additem") { Link Here
707	next if ref $error eq 'Koha::Item'; # Deleted item is returned if deletion successful	709	next if ref $error eq 'Koha::Item'; # Deleted item is returned if deletion successful
708	push @errors,$error;	710	push @errors,$error;
709	}	711	}
710	C4::Biblio::ModZebra( $biblionumber, "specialUpdate", "biblioserver" );	712	my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX });
		713	$indexer->index_records( $biblionumber, "specialUpdate", "biblioserver" );
711	if ( @errors ) {	714	if ( @errors ) {
712	$nextop="additem";	715	$nextop="additem";
713	} else {	716	} else {