Bugzilla – Attachment 24031 Details for
Bug 11096
Koha cannot retrieve big records from Zebra
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 11096: Make use of MARCXML conditional of DOM being set
Bug-11096-Make-use-of-MARCXML-conditional-of-DOM-b.patch (text/plain), 13.96 KB, created by
Tomás Cohen Arazi (tcohen)
on 2014-01-08 18:53:23 UTC
(
hide
)
Description:
Bug 11096: Make use of MARCXML conditional of DOM being set
Filename:
MIME Type:
Creator:
Tomás Cohen Arazi (tcohen)
Created:
2014-01-08 18:53:23 UTC
Size:
13.96 KB
patch
obsolete
>From faba8ad60342a51a7ad2f00bb69ea8382264bf1b Mon Sep 17 00:00:00 2001 >From: Tomas Cohen Arazi <tomascohen@gmail.com> >Date: Wed, 8 Jan 2014 11:27:26 -0300 >Subject: [PATCH] Bug 11096: Make use of MARCXML conditional of DOM being set > >We have a way of knowing the indexing mode that is in use: > > C4::Context->config('zebra_bib_index_mode') >and C4::Context->config('zebra_auth_index_mode') > >this patch makes Koha use that to fallback to the old USMARC >serialization format for retreiving from Zebra in case we >are on a GRS-1 setup. > >To test. Check tests now pass: > > $ prove -v t/db_dependent/Search.t > >Sponsored-by: Universidad Nacional de Cordoba >--- > C4/AuthoritiesMarc.pm | 20 ++++++++++++++++---- > C4/Context.pm | 36 +++++++++++++++++------------------- > C4/Search.pm | 49 +++++++++++++++++++++++++++++++++++-------------- > t/db_dependent/Search.t | 42 +++++++++++++++++++++++++++++++++++++++++- > 4 files changed, 109 insertions(+), 38 deletions(-) > >diff --git a/C4/AuthoritiesMarc.pm b/C4/AuthoritiesMarc.pm >index b1d8405..f6b7262 100644 >--- a/C4/AuthoritiesMarc.pm >+++ b/C4/AuthoritiesMarc.pm >@@ -267,10 +267,22 @@ sub SearchAuthorities { > ##Here we have to extract MARC record and $authid from ZEBRA AUTHORITIES > my $rec=$oAResult->record($counter); > my $separator=C4::Context->preference('authoritysep'); >- my $authrecord = MARC::Record->new_from_xml( >- $rec->raw(), >- 'UTF-8' >- ); >+ my $authrecord = eval { >+ if (C4::Context->config('zebra_auth_index_mode') eq 'dom') { >+ MARC::Record->new_from_xml( >+ $rec->raw(), >+ 'UTF-8' >+ ); >+ } else { >+ MARC::Record->new_from_usmarc($rec->raw()); >+ } >+ }; >+ >+ if ( !defined $authrecord or !defined $authrecord->field('001') ) { >+ $counter++; >+ next; >+ } >+ > my $authid=$authrecord->field('001')->data(); > my %newline; > $newline{authid} = $authid; >diff --git a/C4/Context.pm b/C4/Context.pm >index d7289ba..65aaf9a 100644 >--- a/C4/Context.pm >+++ b/C4/Context.pm >@@ -729,14 +729,28 @@ sub _new_Zconn { > > my $tried=0; # first attempt > my $Zconn; # connection object >+ my $elementSetName; >+ > $server //= "biblioserver"; > $syntax //= "XML"; > >+ if ( ($server eq 'biblioserver' and $context->{'config'}->{'zebra_bib_index_mode'} eq 'grs1') >+ or ($server eq 'authorityserver' and $context->{'config'}->{'zebra_auth_index_mode'} eq 'grs1')) { >+ >+ $elementSetName = 'F'; >+ $syntax = 'usmarc'; >+ >+ } else { >+ >+ $elementSetName = 'marcxml'; >+ $syntax = 'XML'; >+ } >+ > my $host = $context->{'listen'}->{$server}->{'content'}; > my $servername = $context->{"config"}->{$server}; > my $user = $context->{"serverinfo"}->{$server}->{"user"}; > my $password = $context->{"serverinfo"}->{$server}->{"password"}; >- $auth = 1 if($user && $password); >+ $auth = 1 if($user && $password); > retry: > eval { > # set options >@@ -748,7 +762,7 @@ sub _new_Zconn { > $o->option(cqlfile=> $context->{"server"}->{$server}->{"cql2rpn"}); > $o->option(cclfile=> $context->{"serverinfo"}->{$server}->{"ccl2rpn"}); > $o->option(preferredRecordSyntax => $syntax); >- $o->option(elementSetName => "marcxml"); >+ $o->option(elementSetName => $elementSetName); > $o->option(databaseName => ($servername?$servername:"biblios")); > > # create a new connection object >@@ -763,23 +777,7 @@ sub _new_Zconn { > } > > }; >-# if ($@) { >-# # Koha manages the Zebra server -- this doesn't work currently for me because of permissions issues >-# # Also, I'm skeptical about whether it's the best approach >-# warn "problem with Zebra"; >-# if ( C4::Context->preference("ManageZebra") ) { >-# if ($@->code==10000 && $tried==0) { ##No connection try restarting Zebra >-# $tried=1; >-# warn "trying to restart Zebra"; >-# my $res=system("zebrasrv -f $ENV{'KOHA_CONF'} >/koha/log/zebra-error.log"); >-# goto "retry"; >-# } else { >-# warn "Error ", $@->code(), ": ", $@->message(), "\n"; >-# $Zconn="error"; >-# return $Zconn; >-# } >-# } >-# } >+ > return $Zconn; > } > >diff --git a/C4/Search.pm b/C4/Search.pm >index 11f0ba4..c108323 100644 >--- a/C4/Search.pm >+++ b/C4/Search.pm >@@ -144,10 +144,19 @@ sub FindDuplicate { > my @results; > if (!defined $error) { > foreach my $possible_duplicate_record (@{$searchresults}) { >- my $marcrecord = MARC::Record->new_from_xml( >- $possible_duplicate_record, >- 'UTF-8' >- ); >+ my $marcrecord = eval { >+ if ( C4::Context->config('zebra_bib_index_mode') eq 'dom') { >+ MARC::Record->new_from_xml( >+ $possible_duplicate_record, >+ 'UTF-8' >+ ); >+ } else { >+ MARC::Record->new_from_usmarc( >+ $possible_duplicate_record >+ ); >+ } >+ }; >+ > my $result = TransformMarcToKoha( $dbh, $marcrecord, '' ); > > # FIXME :: why 2 $biblionumber ? >@@ -291,10 +300,11 @@ sub SimpleSearch { > } > > for my $j ( $first_record .. $last_record ) { >- my $record = >+ my $record = eval { > $tmpresults[ $i - 1 ]->record( $j - 1 )->raw() > ; # 0 indexed >- push @{$results}, $record; >+ }; >+ push @{$results}, $record if defined $record; > } > } > ); >@@ -506,10 +516,18 @@ sub getRecords { > for my $facet (@$facets) { > for ( my $j = 0 ; $j < $jmax ; $j++ ) { > >- my $marc_record = eval { MARC::Record->new_from_xml( >- $results[ $i - 1 ]->record($j)->raw(), >- 'UTF-8' >- ); }; >+ my $marc_record = eval { >+ if ( C4::Context->config('zebra_bib_index_mode') eq 'dom' ) { >+ MARC::Record->new_from_xml( >+ $results[ $i - 1 ]->record($j)->raw(), >+ 'UTF-8' >+ ); >+ } else { >+ MARC::Record->new_from_usmarc( >+ $results[ $i - 1 ]->record($j)->raw() >+ ); >+ } >+ }; > > if ( $@ ) { > warn "ERROR DECODING RECORD - $@: " . >@@ -1708,10 +1726,13 @@ sub searchResults { > > # loop through all of the records we've retrieved > for ( my $i = $offset ; $i <= $times - 1 ; $i++ ) { >- my $marcrecord = eval { MARC::Record->new_from_xml( >- $marcresults->[$i], >- 'UTF-8' >- ); }; >+ my $marcrecord = eval { >+ if ( C4::Context->config('zebra_bib_index_mode') eq 'dom') { >+ MARC::Record->new_from_xml( $marcresults->[$i], 'UTF-8' ); >+ } else { >+ MARC::Record->new_from_usmarc( $marcresults->[$i] ); >+ } >+ }; > > if ( $@ ) { > warn "ERROR DECODING RECORD - $@: " . $marcresults->[$i]; >diff --git a/t/db_dependent/Search.t b/t/db_dependent/Search.t >index 2907a7d..887bf0f 100644 >--- a/t/db_dependent/Search.t >+++ b/t/db_dependent/Search.t >@@ -359,6 +359,7 @@ sub run_marc21_search_tests { > getRecords('au:Lessig', 'au:Lessig', [], [ 'biblioserver' ], '20', 0, undef, \%branches, \%itemtypes, 'ccl', undef); > is($results_hashref->{biblioserver}->{hits}, 4, "getRecords title search for 'Australia' matched right number of records"); > >+if ( $indexing_mode eq 'dom' ) { > ( undef, $results_hashref, $facets_loop ) = > getRecords('salud', 'salud', [], [ 'biblioserver' ], '19', 0, undef, \%branches, \%itemtypes, 'ccl', undef); > ok(MARC::Record::new_from_xml($results_hashref->{biblioserver}->{RECORDS}->[0],'UTF-8')->title_proper() =~ m/^Efectos del ambiente/ && >@@ -393,6 +394,44 @@ sub run_marc21_search_tests { > MARC::Record::new_from_xml($results_hashref->{biblioserver}->{RECORDS}->[7],'UTF-8')->title_proper() eq 'World health statistics 2009^ien' && > MARC::Record::new_from_xml($results_hashref->{biblioserver}->{RECORDS}->[18],'UTF-8')->title_proper() eq 'Manual de higiene industrial^ies' > , "Simple descending publication date sorting in getRecords matches old behavior"); >+ >+} elsif ( $indexing_mode eq 'grs1' ){ >+ ( undef, $results_hashref, $facets_loop ) = >+ getRecords('salud', 'salud', [], [ 'biblioserver' ], '19', 0, undef, \%branches, \%itemtypes, 'ccl', undef); >+ ok(MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[0])->title_proper() =~ m/^Efectos del ambiente/ && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[7])->title_proper() eq 'Salud y seguridad de los trabajadores del sector salud: manual para gerentes y administradores^ies' && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[18])->title_proper() =~ m/^Indicadores de resultados identificados/ >+ , "Simple relevance sorting in getRecords matches old behavior"); >+ >+ ( undef, $results_hashref, $facets_loop ) = >+ getRecords('salud', 'salud', [ 'author_az' ], [ 'biblioserver' ], '38', 0, undef, \%branches, \%itemtypes, 'ccl', undef); >+ ok(MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[0])->title_proper() =~ m/la enfermedad laboral\^ies$/ && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[6])->title_proper() =~ m/^Indicadores de resultados identificados/ && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[18])->title_proper() eq 'World health statistics 2009^ien' >+ , "Simple ascending author sorting in getRecords matches old behavior"); >+ >+ ( undef, $results_hashref, $facets_loop ) = >+ getRecords('salud', 'salud', [ 'author_za' ], [ 'biblioserver' ], '38', 0, undef, \%branches, \%itemtypes, 'ccl', undef); >+ ok(MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[0])->title_proper() eq 'World health statistics 2009^ien' && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[12])->title_proper() =~ m/^Indicadores de resultados identificados/ && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[18])->title_proper() =~ m/la enfermedad laboral\^ies$/ >+ , "Simple descending author sorting in getRecords matches old behavior"); >+ >+ ( undef, $results_hashref, $facets_loop ) = >+ getRecords('salud', 'salud', [ 'pubdate_asc' ], [ 'biblioserver' ], '38', 0, undef, \%branches, \%itemtypes, 'ccl', undef); >+ ok(MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[0])->title_proper() eq 'Manual de higiene industrial^ies' && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[7])->title_proper() =~ m/seguridad e higiene del trabajo\^ies$/ && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[18])->title_proper() =~ m/^Indicadores de resultados identificados/ >+ , "Simple ascending publication date sorting in getRecords matches old behavior"); >+ >+ ( undef, $results_hashref, $facets_loop ) = >+ getRecords('salud', 'salud', [ 'pubdate_dsc' ], [ 'biblioserver' ], '38', 0, undef, \%branches, \%itemtypes, 'ccl', undef); >+ ok(MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[0])->title_proper() =~ m/^Estado de salud/ && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[7])->title_proper() eq 'World health statistics 2009^ien' && >+ MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[18])->title_proper() eq 'Manual de higiene industrial^ies' >+ , "Simple descending publication date sorting in getRecords matches old behavior"); >+} >+ > TODO: { > local $TODO = "Switch relevance search to MARCXML too"; > ( undef, $results_hashref, $facets_loop ) = >@@ -402,6 +441,7 @@ TODO: { > is($record->subfield('100', 'a'), 2, "Scan returned correct number of records matching term"); > # Time to test buildQuery and searchResults too. > } >+ > my ( $query, $simple_query, $query_cgi, > $query_desc, $limit, $limit_cgi, $limit_desc, > $stopwords_removed, $query_type ); >@@ -541,7 +581,7 @@ TODO: { > ($error, $results_hashref, $facets_loop) = getRecords($query,$simple_query,[ ], [ 'biblioserver' ],20,0,undef,\%branches,\%itemtypes,$query_type,0); > is($results_hashref->{biblioserver}->{hits}, 19, "Weighted query returned correct number of results"); > if ($indexing_mode eq 'grs1') { >- is(MARC::Record::new_from_xml($results_hashref->{biblioserver}->{RECORDS}->[0],'UTF-8')->title_proper(), 'Salud y seguridad de los trabajadores del sector salud: manual para gerentes y administradores^ies', "Weighted query returns best match first"); >+ is(MARC::Record::new_from_usmarc($results_hashref->{biblioserver}->{RECORDS}->[0])->title_proper(), 'Salud y seguridad de los trabajadores del sector salud: manual para gerentes y administradores^ies', "Weighted query returns best match first"); > } else { > local $TODO = "Query weighting does not behave exactly the same in DOM vs. GRS"; > is(MARC::Record::new_from_xml($results_hashref->{biblioserver}->{RECORDS}->[0],'UTF-8')->title_proper(), 'Salud y seguridad de los trabajadores del sector salud: manual para gerentes y administradores^ies', "Weighted query returns best match first"); >-- >1.8.3.2
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 11096
:
22291
|
22292
|
22821
|
22845
|
22846
|
22997
|
23030
|
23033
|
23234
|
23235
|
23236
|
23237
|
23336
|
23337
|
23366
|
24019
|
24020
|
24029
|
24030
|
24031
|
24172
|
24186
|
24187
|
24188
|
24490
|
24491
|
24492
|
24580
|
24602
|
24603
|
24604
|
24605
|
24607
|
24731
|
24732
|
24874
|
24894
|
24895
|
24896
|
24901
|
25282
|
25634
|
25635
|
25636