From 1852990e8321625e6d9a563030d5c6cc3630d088 Mon Sep 17 00:00:00 2001 From: wajasu Date: Tue, 8 Apr 2014 21:53:17 -0500 Subject: [PATCH] Bug 11232 - support zebra facet queries in search Minimal support for zebra facet queries with indexes being made available in the zidx fields added to the Koha::getFacets routine. Search falls back to current implementation when facet query has and error or no results. This enables folks to roll out index changes. Currently supports the existing facets. Test Plan: 1) Setup a working git clone dev install with a database with biblios (be sure to have your KOHA_CONF and PERL5LIB set. dom, icu 2) Run searches with current facet behavior (keeping browser tab for later reference) (use staff client to find some biblios and on the marc tab write down the biblionumber for later use. get about 5 or more) (enable syspref for showing facet counters in staff client admin) 3) (Optional) Setup yaz-client for manual validation (on a console tab) yaz-client unix:/home/koha/koha-dev/var/run/zebradb/bibliosocket base biblios form xml elem zebra::facet::Author:0 f @attrset Bib-1 @attr 1=1016 @attr 4=6 @attr 5=1 "Ferguson, Sinclair B." ... some hits s (should show results when thedesignated index Author:0 is specified) elem Subject-to:0,Subject-geo:0,Subject-ut:0,Author:0,Title-series:0,itype:0,location:0,holdingbranch:0 s (should show all facets results if all are configured ) 4) Apply patch 5) cd to your chosen .../koha-dev/etc/zebradb/marc_defs/marc21/biblios (on another console tab) 6) xsltproc ../../../xsl/koha-indexdefs-to-zebra.xsl biblio-koha-indexdefs.xml > biblio-zebra-indexdefs.xsl 7) from kohaclone dir (setup this up in another console tab for steps 7 & 8) run misc/migration_tools/rebuild_zebra.pl --where "biblionumber in (n1, n1, ...)" -b -r --run-as-root -v where n1 ... are the biblios you write down above. (For a quick index turnaround) For a full database rebuild, that will use the new biblio-zebra-indexdefs.xsl from kohaclone dir run misc/migration_tools/rebuild_zebra.pl -a -b -r --run-as-root -v (--run-as-root if needed) 8) zebrasrv -f /home/koha/koha-dev/etc/koha-conf.xml (can watch this on output and cut RPN queries for yaz if you desire) 9) use browser to search as you did in step 2 to see if you have larger counters (when you have a bigger database/result) 10) if facets are satisfactory, we pass. please report anomalies. You can edit the C4/Search.pm line that defines $element_set_all_indexes and make adjustments removing selectively,save and then do a search to see things fallback to the current code behavior. Write now this patch demonstrates this strategy as a baby step for a release. Things that need to be done: - clean code (warnings, comments, long/bad var names, etc) - add facet usage syspref detection (if needed) - pubdate, and other desired additions (can be a future/followup enhancment) - think about UTF-8 and check if diacritics etc show in facet term - discuss naming of some added indexes Subject-to vs su-to, Subject-geo vs su-geo - handle HideLostItems, and check if Suppress is handled. - grs1 and other adjustments (if needed) - future enhancment can: support syspref/editing of element_indexes_all and the order to display - communicate the adjustment to biblio-koha-indexdefs.xml for upgrades to take advantage. - if this patch strategy is acceptable, the UNIMARC and NORMARC facets can be worked on. fallback allows us to rollout. - no restrictions on the total amount of facets are configured (Author:0:100 might be worth trying) - facet consolidation for the presentation UI is the same but could return all results to the UI and let CSS hide/unhide. (another bug exists for this) I did NYTProf, but others can and suggest performance enhancements. When zebra facet indexes are available no marc records are being read etc, as of now, unless things fallback. Note: Title-series:0 seemed to not show a 490a in a certain case where I had 10 results with a certain resultset. Weird, but me might just decide to fallback for that index if its an issue. --- C4/Koha.pm | 20 ++++ C4/Search.pm | 121 ++++++++++++++++++++- .../marc21/biblios/biblio-koha-indexdefs.xml | 47 +++++--- 3 files changed, 174 insertions(+), 14 deletions(-) diff --git a/C4/Koha.pm b/C4/Koha.pm index 4202957..3aa2895 100644 --- a/C4/Koha.pm +++ b/C4/Koha.pm @@ -696,12 +696,16 @@ sub getFacets { label => 'Titles', tags => [ qw/ 500a 501a 503a / ], sep => ', ', + zidx => 'Title', + zreg => '0', }, { idx => 'au', label => 'Authors', tags => [ qw/ 700ab 701ab 702ab / ], sep => C4::Context->preference("UNIMARCAuthorsFacetsSeparator"), + zidx => 'Author', + zreg => '0', }, { idx => 'se', @@ -733,6 +737,8 @@ sub getFacets { label => 'Topics', tags => [ qw/ 650a / ], sep => '--', + zidx => 'Subject-to', + zreg => '0', }, # { # idx => 'su-na', @@ -745,35 +751,47 @@ sub getFacets { label => 'Places', tags => [ qw/ 651a / ], sep => '--', + zidx => 'Subject-geo', + zreg => '0', }, { idx => 'su-ut', label => 'Titles', tags => [ qw/ 630a / ], sep => '--', + zidx => 'Subject', + zreg => '0', }, { idx => 'au', label => 'Authors', tags => [ qw/ 100a 110a 700a / ], sep => ', ', + zidx => 'Author', + zreg => '0', }, { idx => 'se', label => 'Series', tags => [ qw/ 440a 490a / ], sep => ', ', + zidx => 'Title-series', + zreg => '0', }, { idx => 'itype', label => 'ItemTypes', tags => [ qw/ 952y 942c / ], sep => ', ', + zidx => 'itype', + zreg => '0', }, { idx => 'location', label => 'Location', tags => [ qw / 952c / ], + zidx => 'location', + zreg => '0', }, ]; @@ -783,6 +801,8 @@ sub getFacets { idx => 'branch', label => 'Libraries', tags => [ qw / 952b / ], + zidx => 'holdingbranch', + zreg => '0', }; } push( @$facets, $library_facet ); diff --git a/C4/Search.pm b/C4/Search.pm index 09c1951..7a6c878 100644 --- a/C4/Search.pm +++ b/C4/Search.pm @@ -39,6 +39,8 @@ use MARC::Field; use utf8; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $DEBUG); +use Data::Dumper; + # set the version for version checking BEGIN { $VERSION = 3.07.00.049; @@ -346,6 +348,10 @@ sub getRecords { my $facets = getFacets(); my $facets_maxrecs = C4::Context->preference('maxRecordsForFacets')||20; + my $zxpc = XML::LibXML::XPathContext->new; + $zxpc->registerNs('z', 'http://www.indexdata.com/zebra/'); + + my @facets_results; # stores rsults from facets query for corresponding server query (biblioserver only) my @facets_loop; # stores the ref to array of hashes for template facets loop ### LOOP THROUGH THE SERVERS @@ -450,7 +456,6 @@ sub getRecords { else { $times = $size; } - for ( my $j = $offset ; $j < $times ; $j++ ) { my $records_hash; my $record; @@ -501,11 +506,121 @@ sub getRecords { $results_hashref->{ $servers[ $i - 1 ] } = $results_hash; # Fill the facets while we're looping, but only for the biblioserver and not for a scan + + if ( !$scan && $servers[ $i - 1 ] =~ /biblioserver/ ) { + # query for all the facets using zebra::facet and index :0 setup for all desired facets at once + #TODO: build up element_set_all_indexes from getFacets @$facets data structure OR syspref if available + my $element_set_all_indexes = 'Subject-to:0,Subject-geo:0,Subject-ut:0,Author:0,Title-series:0,itype:0,location:0,holdingbranch:0'; + my $element_set_all = 'zebra::facet::'.$element_set_all_indexes; + my $saved_zoptions = { "preferredRecordSyntax" => $results[$i - 1]->option("preferredRecordSyntax"), + "elementSetName" => $results[$i - 1]->option("elementSetName") + }; + $results[ $i -1 ]->option(preferredRecordSyntax => 'XML'); + $results[ $i -1 ]->option(elementSetName => $element_set_all); + my $rec = $results[ $i -1 ]->record(0); +# warn "Found ", $results[ $i -1 ]->size(), " records\n"; + # If the elementSetName is not configured in the index configs we will get + # ERROR 25 - ZOOM error 25 "Specified element set name not valid for specified database" (addinfo: "F") from diag-set 'Bib-1' + # and $rec will be undefined + + my $facets_records_xml; + if (defined $rec) { + $facets_records_xml = $rec->raw(); #$results[ $i - 1 ]->record(1)->raw(); + warn "FACET_RECORDS :".$facets_records_xml; + + if ($rec->error()) { + my($code, $msg, $addinfo, $dset) = $rec->error(); + warn "error $code, $msg ($addinfo) from $dset set\n"; + #die $rec->exception(); + } +# warn Dumper( $rec ); + } + + my $dom; + my $parser = XML::LibXML->new( ext_ent_handler => sub { die "External entities are not supported\n"; }); + if ($facets_records_xml) { + # parse the query results posibly specifiy encoding utf-8 + $dom = $parser->parse_string($facets_records_xml); + } else { + # query each facet index individually, building up a cumulative facet DOM xml + # This will make things more robust in case one index mentioned in element_set above is missing + # OR if the index is not present when none of the results has that facet. + + my @zf_indexes = split(',', $element_set_all_indexes); + for my $element_index (@zf_indexes) { + my $element_set = 'zebra::facet::'.$element_index; +# warn "elem set: $element_set"; + $results[ $i -1 ]->option(elementSetName => $element_set); + my $rec_chunk = $results[ $i -1 ]->record(0); + # If the elementSetName is not configured in the index configs we will get + # ERROR 25 - ZOOM error 25 "Specified element set name not valid for specified database" (addinfo: "F") from diag-set 'Bib-1' + # and $rec_chunk will be undefined + my $facets_records_xml_chunk; + if (defined $rec_chunk) { + $facets_records_xml_chunk = $rec_chunk->raw(); #$results[ $i - 1 ]->record(1)->raw(); +# warn "FACET_RECORDS_CHUNK :".$facets_records_xml_chunk if ($facets_records_xml_chunk); + + if ($rec_chunk->error()) { + my($code, $msg, $addinfo, $dset) = $rec_chunk->error(); + warn "error $code, $msg ($addinfo) from $dset set\n"; + #die $rec->exception(); + next; # skip this facet since no index available + } + + if (!$facets_records_xml && $facets_records_xml_chunk) { + $facets_records_xml = substr $facets_records_xml_chunk, 0 , -10; # drop + # warn "FRX:".$facets_records_xml; + # $facets_records_xml = $facets_records_xml_chunk; + } else { + $facets_records_xml_chunk = substr $facets_records_xml_chunk, index($facets_records_xml_chunk,''; +#warn "FRXXXXX:".$facets_records_xml; + $dom = $parser->parse_string($facets_records_xml); + } + } + + # restore connection settings for later result queries + $results[ $i -1 ]->option(preferredRecordSyntax => $saved_zoptions->{"preferredRecordSyntax"}); + $results[ $i -1 ]->option(elementSetName => $saved_zoptions->{"elementSetName"}); my $jmax = $size > $facets_maxrecs ? $facets_maxrecs : $size; for my $facet (@$facets) { + my @facet_nodes = (); + if (defined $facet->{zidx} && + (@facet_nodes = $zxpc->findnodes('//z:facet[@index="'.$facet->{zidx}.'"]',$dom)) ) { +#warn "ZIDX ".$facet->{zidx}; +#my $fctcnt = @facet_nodes; +#warn "fctcnt=$fctcnt"; + my $fnode = @facet_nodes[0]; # should be one facet node for this term + my $tnodes = $zxpc->findnodes('z:term',$fnode); # array of elements "terms" for this facet node +#my $tfncnt = $tnodes->size; +#warn "tfncnt=$tfncnt"; + for ( my $j = 0 ; $j < $tnodes->size() && $j < $jmax ; $j++ ) { + # query zebra for this indexes facets for up to N max facet records + # done above so use xpath against the dom above + + my $tfn = $tnodes->get_node($j); # should be Nth term +#warn "ntype=$ntype"; + my $occur = $tfn->getAttribute('occur'); +#warn "occur=$occur"; + my $tfntext = $tfn->textContent(); +#warn "tfntext=$tfntext"; + # TODO: could examine if "normalized" term matches to prior and skip this one + + $facets_counter->{ $facet->{idx} }->{$tfntext} = $occur; + } + + } else { # build facet from marc + for ( my $j = 0 ; $j < $jmax ; $j++ ) { my $marc_record = new_record_from_zebra ( @@ -534,12 +649,16 @@ sub getRecords { my $data = $field->as_string( $subfield_letters, $facet->{sep} ); unless ( $data ~~ @used_datas ) { +#warn "TermData: $data"; push @used_datas, $data; $facets_counter->{ $facet->{idx} }->{$data}++; } } # fields } # field codes } # records + + } + $facets_info->{ $facet->{idx} }->{label_value} = $facet->{label}; $facets_info->{ $facet->{idx} }->{expanded} = diff --git a/etc/zebradb/marc_defs/marc21/biblios/biblio-koha-indexdefs.xml b/etc/zebradb/marc_defs/marc21/biblios/biblio-koha-indexdefs.xml index 5ddbb62..a1c02c2 100644 --- a/etc/zebradb/marc_defs/marc21/biblios/biblio-koha-indexdefs.xml +++ b/etc/zebradb/marc_defs/marc21/biblios/biblio-koha-indexdefs.xml @@ -243,9 +243,8 @@ Author:w - Author:p - Author:s - Editor:w + Author:p Author:0 + Author:s Editor:w Author-personal-bibliography:w Author-personal-bibliography:p Author-personal-bibliography:s @@ -268,7 +267,7 @@ Author:w - Author:p + Author:p Author:0 Author:s Author-title:w Author-name-corporate:w @@ -491,7 +490,7 @@ Title-series:w - Title-series:p + Title-series:p Title-series:0 @@ -502,12 +501,11 @@ Title-series:w Title-series:p Title:w - Title-series:w Title-series:w - Title-series:p + Title-series:p Title-series:0 @@ -644,6 +642,14 @@ Subject:w Subject:p + + + + Subject-ut:0 + Thematic-number:w @@ -665,6 +671,10 @@ Subject:w Subject:p + + + Subject-to:0 + Koha-Auth-Number:w @@ -674,6 +684,10 @@ Subject:w Subject:p + + + Subject-geo:0 + Koha-Auth-Number:w @@ -776,7 +790,7 @@ Author:w - Author:p + Author:p Author:0 @@ -1039,6 +1053,13 @@ Name:w Conference-name:w + + Record-control-number:w @@ -1090,7 +1111,7 @@ itemtype:w itemtype:p itype:w - itype:p + itype:p itype:0 @@ -1163,15 +1184,15 @@ - homebranch:w + homebranch:w homebranch:0 - holdingbranch:w + holdingbranch:w holdingbranch:0 - location:w + location:w location:0 @@ -1259,7 +1280,7 @@ itype:w - itype:p + itype:p itype:0 -- 1.8.5.3