From b4900001788401970c6920efb4e82d636fc40749 Mon Sep 17 00:00:00 2001 From: Jonathan Druart Date: Fri, 29 Oct 2021 11:15:48 +0200 Subject: [PATCH] Bug 29368: Correctly mock Zebra index from tests This is just reusing what is done in Search.t (and that is correct). But search_utf8.t and remove_from_cart.t are wrong as we want to use the UI (and we cannot mock the zebra index, ie. koha-conf, from tests for plack). This still needs some work but improve a bit the existing code and make tests pass (hopefully!) --- t/db_dependent/Search.t | 119 +++++++++------------ t/db_dependent/selenium/remove_from_cart.t | 7 +- t/db_dependent/www/search_utf8.t | 21 ++-- t/lib/Mocks/Zebra.pm | 79 ++++++++++++-- 4 files changed, 132 insertions(+), 94 deletions(-) diff --git a/t/db_dependent/Search.t b/t/db_dependent/Search.t index 8ccb0d81804..6d84995b938 100755 --- a/t/db_dependent/Search.t +++ b/t/db_dependent/Search.t @@ -30,6 +30,7 @@ use Test::More tests => 3; use Test::MockModule; use Test::Warn; use t::lib::Mocks; +use t::lib::Mocks::Zebra; use Koha::Caches; @@ -41,56 +42,9 @@ use File::Find; use File::Temp qw/ tempdir /; use File::Path; -our $child; -our $datadir; - -sub index_sample_records_and_launch_zebra { - my ($datadir, $marc_type) = @_; - - my $sourcedir = dirname(__FILE__) . "/data"; - unlink("$datadir/zebra.log"); - if (-f "$sourcedir/${marc_type}/zebraexport/biblio/exported_records") { - my $zebra_bib_cfg = 'zebra-biblios-dom.cfg'; - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_bib_cfg -v none,fatal -g iso2709 -d biblios init"); - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_bib_cfg -v none,fatal -g iso2709 -d biblios update $sourcedir/${marc_type}/zebraexport/biblio"); - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_bib_cfg -v none,fatal -g iso2709 -d biblios commit"); - } - # ... and add large bib records, if present - if (-f "$sourcedir/${marc_type}/zebraexport/large_biblio/exported_records.xml") { - my $zebra_bib_cfg = 'zebra-biblios-dom.cfg'; - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_bib_cfg -v none,fatal -g marcxml -d biblios update $sourcedir/${marc_type}/zebraexport/large_biblio"); - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_bib_cfg -v none,fatal -g marcxml -d biblios commit"); - } - if (-f "$sourcedir/${marc_type}/zebraexport/authority/exported_records") { - my $zebra_auth_cfg = 'zebra-authorities-dom.cfg'; - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_auth_cfg -v none,fatal -g iso2709 -d authorities init"); - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_auth_cfg -v none,fatal -g iso2709 -d authorities update $sourcedir/${marc_type}/zebraexport/authority"); - system("zebraidx -c $datadir/etc/koha/zebradb/$zebra_auth_cfg -v none,fatal -g iso2709 -d authorities commit"); - } - - $child = fork(); - if ($child == 0) { - exec("zebrasrv -f $datadir/etc/koha-conf.xml -v none,request -l $datadir/zebra.log"); - exit; - } - - sleep(1); -} - -sub cleanup { - if ($child) { - kill 9, $child; - - # Clean up the Zebra files since the child process was just shot - rmtree $datadir; - } -} - # Fall back to make sure that the Zebra process # and files get cleaned up -END { - cleanup(); -} +our @cleanup; sub matchesExplodedTerms { my ($message, $query, @terms) = @_; @@ -211,7 +165,7 @@ our $bibliomodule = Test::MockModule->new('C4::Biblio'); sub mock_GetMarcSubfieldStructure { my $marc_type = shift; - if ($marc_type eq 'marc21') { + if ($marc_type eq 'MARC21') { $bibliomodule->mock('GetMarcSubfieldStructure', sub { return { 'biblio.biblionumber' => [{ tagfield => '999', tagsubfield => 'c' }], @@ -259,14 +213,12 @@ sub mock_GetMarcSubfieldStructure { } sub run_marc21_search_tests { - $datadir = tempdir(); - system(dirname(__FILE__) . "/zebra_config.pl $datadir marc21"); - Koha::Caches->get_instance('config')->flush_all; + $marcflavour = 'MARC21'; + my $mock_zebra = t::lib::Mocks::Zebra->new({marcflavour => $marcflavour}); + push @cleanup, $mock_zebra; - mock_GetMarcSubfieldStructure('marc21'); - my $context = C4::Context->new("$datadir/etc/koha-conf.xml"); - $context->set_context(); + mock_GetMarcSubfieldStructure($marcflavour); use_ok('C4::Search', qw( getIndexes FindDuplicate SimpleSearch getRecords buildQuery searchResults )); @@ -275,7 +227,6 @@ sub run_marc21_search_tests { $QueryAutoTruncate = 0; $QueryWeightFields = 0; $QueryFuzzy = 0; - $marcflavour = 'MARC21'; my $indexes = C4::Search::getIndexes(); is(scalar(grep(/^ti$/, @$indexes)), 1, "Title index supported"); @@ -310,7 +261,23 @@ sub run_marc21_search_tests { 'VM' => { 'imageurl' => 'bridge/dvd.png', 'summary' => '', 'itemtype' => 'VM', 'description' => 'Visual Materials' }, ); - index_sample_records_and_launch_zebra($datadir, 'marc21'); + my $sourcedir = dirname(__FILE__) . "/data"; + $mock_zebra->load_records( + sprintf( "%s/%s/zebraexport/biblio", $sourcedir, lc($marcflavour) ), + 'iso2709', 'biblios', 1 ); + $mock_zebra->load_records( + sprintf( + "%s/%s/zebraexport/large_biblio", + $sourcedir, lc($marcflavour) + ), + 'marcxml', 'biblios', 0 + ); + $mock_zebra->load_records( + sprintf( "%s/%s/zebraexport/authority", $sourcedir, lc($marcflavour) ), + 'iso2709', 'authorities', 1 + ); + + $mock_zebra->launch_zebra; my ($biblionumber, $title); my $record = MARC::Record->new; @@ -830,18 +797,16 @@ ok(MARC::Record::new_from_xml($results_hashref->{biblioserver}->{RECORDS}->[0],' is_deeply( $facets_info, $expected_facets_info_marc21, "_get_facets_info returns the correct data"); - cleanup(); + $mock_zebra->cleanup; } sub run_unimarc_search_tests { - $datadir = tempdir(); - system(dirname(__FILE__) . "/zebra_config.pl $datadir unimarc"); - Koha::Caches->get_instance('config')->flush_all; + $marcflavour = 'UNIMARC'; + my $mock_zebra = t::lib::Mocks::Zebra->new({marcflavour => $marcflavour}); + push @cleanup, $mock_zebra; - mock_GetMarcSubfieldStructure('unimarc'); - my $context = C4::Context->new("$datadir/etc/koha-conf.xml"); - $context->set_context(); + mock_GetMarcSubfieldStructure($marcflavour); use_ok('C4::Search', qw( getIndexes FindDuplicate SimpleSearch getRecords buildQuery searchResults )); @@ -850,9 +815,23 @@ sub run_unimarc_search_tests { $QueryAutoTruncate = 0; $QueryWeightFields = 0; $QueryFuzzy = 0; - $marcflavour = 'UNIMARC'; - index_sample_records_and_launch_zebra($datadir, 'unimarc'); + my $sourcedir = dirname(__FILE__) . "/data"; + $mock_zebra->load_records( + sprintf( "%s/%s/zebraexport/biblio", $sourcedir, lc($marcflavour) ), + 'iso2709', 'biblios', 1 ); + $mock_zebra->load_records( + sprintf( + "%s/%s/zebraexport/large_biblio", + $sourcedir, lc($marcflavour) + ), + 'marcxml', 'biblios', 0 + ); + $mock_zebra->load_records( + sprintf( "%s/%s/zebraexport/authority", $sourcedir, lc($marcflavour) ), + 'iso2709', 'authorities', 1 + ); + $mock_zebra->launch_zebra; my ( $error, $marcresults, $total_hits ) = SimpleSearch("ti=Järnvägarnas efterfrågan och den svenska industrin", 0, 10); is($total_hits, 1, 'UNIMARC title search'); @@ -916,7 +895,7 @@ sub run_unimarc_search_tests { is_deeply( $facets_info, $expected_facets_info_unimarc, "_get_facets_info returns the correct data"); - cleanup(); + $mock_zebra->cleanup; } subtest 'MARC21 + DOM' => sub { @@ -933,8 +912,8 @@ subtest 'UNIMARC + DOM' => sub { subtest 'FindDuplicate' => sub { plan tests => 6; Koha::Caches->get_instance('config')->flush_all; - t::lib::Mocks::mock_preference('marcflavour', 'marc21' ); - mock_GetMarcSubfieldStructure('marc21'); + t::lib::Mocks::mock_preference('marcflavour', 'MARC21' ); + mock_GetMarcSubfieldStructure('MARC21'); my $z_searcher = Test::MockModule->new('C4::Search'); $z_searcher->mock('SimpleSearch', sub { warn shift @_; @@ -980,6 +959,8 @@ subtest 'FindDuplicate' => sub { Koha::Caches->get_instance('config')->flush_all; END { + + $_->cleanup for @cleanup; my $dbh = C4::Context->dbh; # Restore visibility of subfields in OPAC $dbh->do(q{ diff --git a/t/db_dependent/selenium/remove_from_cart.t b/t/db_dependent/selenium/remove_from_cart.t index 02ce186b399..1cefe3178f2 100755 --- a/t/db_dependent/selenium/remove_from_cart.t +++ b/t/db_dependent/selenium/remove_from_cart.t @@ -44,11 +44,8 @@ C4::Context->set_preference('SearchEngine', 'Zebra'); my $mock_zebra = t::lib::Mocks::Zebra->new( { - koha_conf => $ENV{KOHA_CONF}, - user => $ENV{KOHA_USER}, - password => $ENV{KOHA_PASS}, - intranet => $ENV{KOHA_INTRANET_URL}, - opac => $ENV{KOHA_OPAC_URL}, + koha_conf => $ENV{KOHA_CONF}, + marcflavour => $marcflavour, } ); diff --git a/t/db_dependent/www/search_utf8.t b/t/db_dependent/www/search_utf8.t index 4bbfbec8885..70d4e1464c5 100755 --- a/t/db_dependent/www/search_utf8.t +++ b/t/db_dependent/www/search_utf8.t @@ -34,9 +34,6 @@ use t::lib::Mocks::Zebra; my $testdir = File::Spec->rel2abs( dirname(__FILE__) ); # global variables that will be used when forking -my $koha_conf = $ENV{KOHA_CONF}; -my $xml = XMLin($koha_conf); - my $marcflavour = C4::Context->preference('marcflavour') || 'MARC21'; my $file1 = @@ -54,8 +51,6 @@ my $file3 = ? "$testdir/data/unimarcutf8supprec.mrc" : "$testdir/data/marc21utf8supprec.mrc"; -my $user = $ENV{KOHA_USER} || $xml->{config}->{user}; -my $password = $ENV{KOHA_PASS} || $xml->{config}->{pass}; our $intranet = $ENV{KOHA_INTRANET_URL}; our $opac = $ENV{KOHA_OPAC_URL}; @@ -76,11 +71,9 @@ $opac =~ s#/$##; my $mock_zebra = t::lib::Mocks::Zebra->new( { - koha_conf => $koha_conf, - user => $user, - password => $password, intranet => $intranet, - opac => $opac + opac => $opac, + koha_conf => $ENV{KOHA_CONF}, } ); @@ -99,12 +92,12 @@ if ( not defined $mock_zebra->{indexer_pid} ) { our $agent = Test::WWW::Mechanize->new( autocheck => 1 ); $agent->get_ok( "$intranet/cgi-bin/koha/mainpage.pl", 'connect to intranet' ); $agent->form_name('loginform'); -$agent->field( 'password', $password ); -$agent->field( 'userid', $user ); +$agent->field( 'userid', $ENV{KOHA_PASS} ); +$agent->field( 'password', $ENV{KOHA_USER} ); $agent->field( 'branch', '' ); $agent->click_ok( '', 'login to staff interface' ); -my $batch_id = $mock_zebra->load_records($file1); +my $batch_id = $mock_zebra->load_records_ui($file1); my $utf8_reg1 = qr/学協会. μμ/; test_search('Αθήνα', 'deuteros', $utf8_reg1); $mock_zebra->clean_records($batch_id); @@ -119,7 +112,7 @@ $mock_zebra->launch_indexer; if ( not defined $mock_zebra->{indexer_pid} ) { plan skip_all => "Tests skip. Error starting the indexer daemon to do those tests\n"; } -$batch_id = $mock_zebra->load_records($file2); +$batch_id = $mock_zebra->load_records_ui($file2); my $utf8_reg2 = qr/Tòmas/; test_search('Ramòn', 'Tòmas',$utf8_reg2); $mock_zebra->clean_records($batch_id); @@ -134,7 +127,7 @@ $mock_zebra->launch_indexer; if ( not defined $mock_zebra->{indexer_pid} ) { plan skip_all => "Tests skip. Error starting the indexer daemon to do those tests\n"; } -$batch_id = $mock_zebra->load_records($file3); +$batch_id = $mock_zebra->load_records_ui($file3); my $utf8_reg3 = qr/😀/; test_search("𠻺tomasito𠻺", 'A tiny record', $utf8_reg3); $mock_zebra->clean_records($batch_id); diff --git a/t/lib/Mocks/Zebra.pm b/t/lib/Mocks/Zebra.pm index a20cd8dd9d2..62e8b9480b2 100644 --- a/t/lib/Mocks/Zebra.pm +++ b/t/lib/Mocks/Zebra.pm @@ -1,5 +1,20 @@ package t::lib::Mocks::Zebra; +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + use Modern::Perl; use Test::More; use File::Basename qw(dirname ); @@ -8,18 +23,50 @@ use File::Path qw( rmtree ); use JSON qw( decode_json ); use C4::ImportBatch; +=head1 NAME + +t::lib::Mocks::Zebra - Trying to mock zebra index + +IMPORTANT NOTE: This module is not working as you may think it could work. + +It will effectively create a new koha-conf.xml file in a temporary directory with zebra config files correctly. +So it will not affect the koha-conf used by plack (and so the UI). + +If you pass koha_conf to the constructor the usual zebra db will be used, otherwise a new koha-conf.xml file will be generated +and the usual zebra db will not be affected. However you must pass $ENV{KOHA_CONF} if you want to test the UI. + +=cut + + sub new { my ( $class, $params ) = @_; - my $datadir = tempdir();; + my $marcflavour = $params->{marcflavour} ? lc($params->{marcflavour}) : 'marc21'; + my $koha_conf = $params->{koha_conf}; + + my $datadir = tempdir(); + my $zebra_db_dir; + unless ( $koha_conf ) { + system(dirname(__FILE__) . "/../../db_dependent/zebra_config.pl $datadir $marcflavour"); + + Koha::Caches->get_instance('config')->flush_all; + $koha_conf = "$datadir/etc/koha-conf.xml"; + my $context = C4::Context->new($koha_conf); + $context->set_context(); + $zebra_db_dir = "$datadir/etc/koha/zebradb/"; + } else { + $koha_conf = $ENV{KOHA_CONF}; + $zebra_db_dir = dirname($koha_conf); + } + my $self = { datadir => $datadir, - koha_conf => $params->{koha_conf}, - user => $params->{user}, - password => $params->{password}, + koha_conf => $koha_conf, + zebra_db_dir => $zebra_db_dir, intranet => $params->{intranet}, opac => $params->{opac} }; + return bless $self, $class; } @@ -30,6 +77,7 @@ sub launch_zebra { my $datadir = $self->{datadir}; my $koha_conf = $self->{koha_conf}; + unlink("$datadir/zebra.log"); my $zebra_pid = fork(); if ( $zebra_pid == 0 ) { exec("zebrasrv -f $koha_conf -v none,request -l $datadir/zebra.log"); @@ -55,6 +103,25 @@ sub launch_indexer { } sub load_records { + my ( $self, $marc_dir, $marc_format, $record_type, $init ) = @_; + + my $datadir = $self->{datadir}; + my $zebra_cfg = $self->{zebra_db_dir} + . ( $record_type eq 'biblios' + ? '/zebra-biblios-dom.cfg' + : '/zebra-authorities-dom.cfg' ); + + my @cmds; + push @cmds, "zebraidx -c $zebra_cfg -v none,fatal -g $marc_format -d $record_type init" if $init; + push @cmds, "zebraidx -c $zebra_cfg -v none,fatal -g $marc_format -d $record_type update $marc_dir"; + push @cmds, "zebraidx -c $zebra_cfg -v none,fatal -g $marc_format -d $record_type commit"; + + for my $cmd ( @cmds ) { + system($cmd); + } +} + +sub load_records_ui { my ( $self, $file ) = @_; my $jsonresponse; my $cgi_root = $self->{intranet} . '/cgi-bin/koha'; @@ -62,8 +129,8 @@ sub load_records { our $agent = Test::WWW::Mechanize->new( autocheck => 1 ); $agent->get_ok( "$cgi_root/mainpage.pl", 'connect to intranet' ); $agent->form_name('loginform'); - $agent->field( 'password', $self->{password} ); - $agent->field( 'userid', $self->{user} ); + $agent->field( 'userid', $ENV{KOHA_PASS} ); + $agent->field( 'password', $ENV{KOHA_USER} ); $agent->field( 'branch', '' ); $agent->click_ok( '', 'login to staff interface' ); -- 2.25.1