From 1775c01eabcbf2f1ea77e200ca51c76f0b47671b Mon Sep 17 00:00:00 2001 From: Marcel de Rooy Date: Thu, 5 Jun 2014 14:20:37 +0200 Subject: [PATCH] Bug 6536: Include SRU searching in Breeding.pm Content-Type: text/plain; charset=utf-8 This patch makes it possible to include SRU servers in Z3950 search. It adjusts the Z3950Search routine in Breeding module. It also replaces SQL code with DBIx statements in Breeding.pm/Z3950Search and the associated scripts z3950search.pl in cataloguing and acqui. Test plan: Verify if a normal Z3950 search still works in cataloging/acqui. Add a SRU target. (You could just use Koha's port 9998.) Define sru_options like sru=get. Use that target in a Z3950 search in cataloging and acqui. (Import.) Test sru_fields translation by comparing search results between various settings for some of the fields. For instance, leave title empty and after that set it to the title field of your SRU target. Signed-off-by: Giuseppe Angilella --- C4/Breeding.pm | 222 +++++++++++++++++++++++-------------------- acqui/z3950_search.pl | 14 ++- cataloguing/z3950_search.pl | 14 ++- 3 files changed, 139 insertions(+), 111 deletions(-) diff --git a/C4/Breeding.pm b/C4/Breeding.pm index 0534665..2001fa5 100644 --- a/C4/Breeding.pm +++ b/C4/Breeding.pm @@ -27,6 +27,7 @@ use C4::Charset; use MARC::File::USMARC; use C4::ImportBatch; use C4::AuthoritiesMarc; #GuessAuthTypeCode, FindDuplicateAuthority +use Koha::Database; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @@ -135,109 +136,36 @@ sub Z3950Search { my @id= @{$pars->{id}}; my $page= $pars->{page}; my $biblionumber= $pars->{biblionumber}; - my $isbn= $pars->{isbn}; - my $issn= $pars->{issn}; - my $title= $pars->{title}; - my $author= $pars->{author}; - my $dewey= $pars->{dewey}; - my $subject= $pars->{subject}; - my $lccn= $pars->{lccn}; - my $lccall= $pars->{lccall}; - my $controlnumber= $pars->{controlnumber}; - my $srchany= $pars->{srchany}; - my $stdid= $pars->{stdid}; my $show_next = 0; my $total_pages = 0; - my $term; my @results; my @breeding_loop = (); my @oConnection; my @oResult; my @errconn; my $s = 0; - my $query; - my $nterms=0; my $imported=0; - my @serverinfo; #replaces former serverhost, servername, encoding - if ($isbn) { - $term=$isbn; - $query .= " \@attr 1=7 \@attr 5=1 \"$term\" "; - $nterms++; - } - if ($issn) { - $term=$issn; - $query .= " \@attr 1=8 \@attr 5=1 \"$term\" "; - $nterms++; - } - if ($title) { - $query .= " \@attr 1=4 \"$title\" "; - $nterms++; - } - if ($author) { - $query .= " \@attr 1=1003 \"$author\" "; - $nterms++; - } - if ($dewey) { - $query .= " \@attr 1=16 \"$dewey\" "; - $nterms++; - } - if ($subject) { - $query .= " \@attr 1=21 \"$subject\" "; - $nterms++; - } - if ($lccn) { - $query .= " \@attr 1=9 $lccn "; - $nterms++; - } - if ($lccall) { - $query .= " \@attr 1=16 \@attr 2=3 \@attr 3=1 \@attr 4=1 \@attr 5=1 \@attr 6=1 \"$lccall\" "; - $nterms++; - } - if ($controlnumber) { - $query .= " \@attr 1=12 \"$controlnumber\" "; - $nterms++; - } - if($srchany) { - $query .= " \@attr 1=1016 \"$srchany\" "; - $nterms++; - } - if($stdid) { - $query .= " \@attr 1=1007 \"$stdid\" "; - $nterms++; - } - for my $i (1..$nterms-1) { - $query = "\@and " . $query; - } - - my $dbh = C4::Context->dbh; - foreach my $servid (@id) { - my $sth = $dbh->prepare("select * from z3950servers where id=?"); - $sth->execute($servid); - while (my $server = $sth->fetchrow_hashref) { - my $option1= new ZOOM::Options(); - $option1->option( 'async' => 1 ); - $option1->option( 'elementSetName', 'F' ); - $option1->option( 'databaseName', $server->{db} ); - $option1->option( 'user', $server->{userid} ) if $server->{userid}; - $option1->option( 'password', $server->{password} ) if $server->{password}; - $option1->option( 'preferredRecordSyntax', $server->{syntax} ); - $option1->option( 'timeout', $server->{timeout} ) if $server->{timeout}; - $oConnection[$s]= create ZOOM::Connection($option1); - $oConnection[$s]->connect( $server->{host}, $server->{port} ); - $serverinfo[$s]->{host}= $server->{host}; - $serverinfo[$s]->{name}= $server->{servername}; - $serverinfo[$s]->{encd}= $server->{encoding} // "iso-5426"; - $s++; - } ## while fetch - } # foreach - my $nremaining = $s; - - for ( my $z = 0 ; $z < $s ; $z++ ) { - $oResult[$z] = $oConnection[$z]->search_pqf($query); - } + my ( $zquery, $squery ) = _build_query( $pars ); + my $schema = Koha::Database->new()->schema(); + my $rs = $schema->resultset('Z3950server')->search( + { id => [ @id ] }, + { result_class => 'DBIx::Class::ResultClass::HashRefInflator' }, + ); + my @servers = $rs->all; + foreach my $server ( @servers ) { + $oConnection[$s] = _create_connection( $server ); + $oResult[$s] = + $server->{servertype} eq 'zed'? + $oConnection[$s]->search_pqf( $zquery ): + $oConnection[$s]->search(new ZOOM::Query::CQL( + _translate_query( $server, $squery ))); + $s++; + } + + my $nremaining = $s; while ( $nremaining-- ) { my $k; my $event; @@ -251,7 +179,7 @@ sub Z3950Search { my ($error)= $oConnection[$k]->error_x(); #ignores errmsg, addinfo, diagset if ($error) { if ($error =~ m/^(10000|10007)$/ ) { - push(@errconn, { server => $serverinfo[$k]->{host}, error => $error } ); + push(@errconn, { server => $servers[$k]->{host}, error => $error } ); } } else { @@ -263,11 +191,11 @@ sub Z3950Search { $total_pages = int($numresults/20)+1 if $total_pages < ($numresults/20); for ($i = ($page-1)*20; $i < (($numresults < ($page*20)) ? $numresults : ($page*20)); $i++) { if($oResult[$k]->record($i)) { - my $res=_handle_one_result($oResult[$k]->record($i), $serverinfo[$k], ++$imported, $biblionumber); #ignores error in sequence numbering + my $res=_handle_one_result($oResult[$k]->record($i), $servers[$k], ++$imported, $biblionumber); #ignores error in sequence numbering push @breeding_loop, $res if $res; } else { - push(@breeding_loop,{'server'=>$serverinfo[$k]->{name},'title'=>join(': ',$oConnection[$k]->error_x()),'breedingid'=>-1,'biblionumber'=>-1}); + push(@breeding_loop,{'server'=>$servers[$k]->{servername},'title'=>join(': ',$oConnection[$k]->error_x()),'breedingid'=>-1,'biblionumber'=>-1}); } } } #if $numresults @@ -289,10 +217,6 @@ sub Z3950Search { $oConnection[$_]->destroy(); } - my @servers = (); - foreach my $id (@id) { - push @servers, {id => $id}; - } $template->param( breeding_loop => \@breeding_loop, servers => \@servers, @@ -300,14 +224,53 @@ sub Z3950Search { ); } +sub _build_query { + my ( $pars ) = @_; + + my $qry_build = { + isbn => '@attr 1=7 @attr 5=1 "#term" ', + issn => '@attr 1=8 @attr 5=1 "#term" ', + title => '@attr 1=4 #term ', + author => '@attr 1=1003 "#term" ', + dewey => '@attr 1=16 "#term" ', + subject => '@attr 1=21 "#term" ', + lccall => '@attr 1=16 @attr 2=3 @attr 3=1 @attr 4=1 @attr 5=1 '. + '@attr 6=1 "#term" ', + controlnumber => '@attr 1=12 "#term" ', + srchany => '@attr 1=1016 "#term" ', + stdid => '@attr 1=1007 "#term" ', + }; + + my $zquery=''; + my $squery=''; + my $nterms=0; + foreach my $k ( keys %$pars ) { + if( ( my $val=$pars->{$k} ) && $qry_build->{$k} ) { + $qry_build->{$k} =~ s/#term/$val/g; + $zquery .= $qry_build->{$k}; + $squery .= "[$k]=\"$val\" and "; + $nterms++; + } + } + $zquery = "\@and " . $zquery for 2..$nterms; + $squery =~ s/ and $//; + return ( $zquery, $squery ); +} + sub _handle_one_result { my ($zoomrec, $servhref, $seq, $bib)= @_; my $raw= $zoomrec->raw(); - my ($marcrecord) = MarcToUTF8Record($raw, C4::Context->preference('marcflavour'), $servhref->{encd}); #ignores charset return values + my $marcrecord; + if( $servhref->{servertype} eq 'sru' ) { + $marcrecord= MARC::Record->new_from_xml( $raw, 'UTF-8', + $servhref->{syntax} ); + } else { + ($marcrecord) = MarcToUTF8Record($raw, C4::Context->preference('marcflavour'), $servhref->{encoding} // "iso-5426" ); #ignores charset return values + } SetUTF8Flag($marcrecord); - my $batch_id = GetZ3950BatchId($servhref->{name}); + my $batch_id = GetZ3950BatchId($servhref->{servername}); my $breedingid = AddBiblioToBatch($batch_id, $seq, $marcrecord, 'UTF-8', 0, 0); #FIXME passing 0 for z3950random #Will eliminate this unused field in a followup report @@ -319,7 +282,7 @@ sub _handle_one_result { return _add_rowdata( { biblionumber => $bib, - server => $servhref->{name}, + server => $servhref->{servername}, breedingid => $breedingid, }, $marcrecord) if $breedingid; } @@ -354,6 +317,63 @@ sub _isbn_replace { return $isbn; } +sub _create_connection { + my ( $server ) = @_; + my $option1= new ZOOM::Options(); + $option1->option( 'async' => 1 ); + $option1->option( 'elementSetName', 'F' ); + $option1->option( 'preferredRecordSyntax', $server->{syntax} ); + $option1->option( 'timeout', $server->{timeout} ) if $server->{timeout}; + + if( $server->{servertype} eq 'sru' ) { + foreach( split ',', $server->{sru_options}//'' ) { + my @temp= split '='; + $option1->option( $temp[0] => $temp[1] ) if @temp; + } + } elsif( $server->{servertype} eq 'zed' ) { + $option1->option( 'databaseName', $server->{db} ); + $option1->option( 'user', $server->{userid} ) if $server->{userid}; + $option1->option( 'password', $server->{password} ) if $server->{password}; + } + + my $obj= ZOOM::Connection->create($option1); + if( $server->{servertype} eq 'sru' ) { + my $host= $server->{host}; + if( $host !~ /^https?:\/\// ) { + #Normally, host will not be prefixed by protocol. + #In that case we can (safely) assume http. + #In case someone prefixed with https, give it a try.. + $host = 'http://' . $host; + } + $obj->connect( $host.':'.$server->{port}.'/'.$server->{db} ); + } else { + $obj->connect( $server->{host}, $server->{port} ); + } + return $obj; +} + +sub _translate_query { #SRU query adjusted per server cf. srufields column + my ($server, $query) = @_; + + #sru_fields is in format title=field,isbn=field,... + #if a field doesn't exist, try anywhere or remove [field]= + my @parts= split(',', $server->{sru_fields} ); + my %trans= map { if( /=/ ) { ( $`,$' ) } else { () } } @parts; + my $any= $trans{srchany}?$trans{srchany}.'=':''; + + my $q=$query; + foreach my $key (keys %trans) { + my $f=$trans{$key}; + if( $f ) { + $q=~s/\[$key\]/$f/g; + } else { + $q=~s/\[$key\]=/$any/g; + } + } + $q=~s/\[\w+\]=/$any/g; # remove remaining fields (not found in field list) + return $q; +} + =head2 ImportBreedingAuth ImportBreedingAuth($marcrecords,$overwrite_auth,$filename,$encoding,$z3950random,$batch_type); diff --git a/acqui/z3950_search.pl b/acqui/z3950_search.pl index 6032b12..f87354f 100755 --- a/acqui/z3950_search.pl +++ b/acqui/z3950_search.pl @@ -31,7 +31,6 @@ use C4::Koha; use C4::Bookseller qw/ GetBookSellerFromId /; my $input = new CGI; -my $dbh = C4::Context->dbh; my $biblionumber = $input->param('biblionumber')||0; my $frameworkcode = $input->param('frameworkcode')||''; my $title = $input->param('title'); @@ -92,11 +91,16 @@ $template->param( ); if ( $op ne "do_search" ) { - my $sth = $dbh->prepare("select id,host,servername,checked from z3950servers where recordtype='biblio' and servertype='zed' order by host"); - $sth->execute(); - my $serverloop = $sth->fetchall_arrayref( {} ); + my $schema = Koha::Database->new()->schema(); + my $rs = $schema->resultset('Z3950server')->search( + { + recordtype => 'biblio', + servertype => ['zed', 'sru'], + }, + { result_class => 'DBIx::Class::ResultClass::HashRefInflator' }, + ); $template->param( - serverloop => $serverloop, + serverloop => [ $rs->all ], opsearch => "search", ); output_html_with_http_headers $input, $cookie, $template->output; diff --git a/cataloguing/z3950_search.pl b/cataloguing/z3950_search.pl index a4afdcc..22dc565 100755 --- a/cataloguing/z3950_search.pl +++ b/cataloguing/z3950_search.pl @@ -29,7 +29,6 @@ use C4::Breeding; use C4::Koha; my $input = new CGI; -my $dbh = C4::Context->dbh; my $error = $input->param('error'); my $biblionumber = $input->param('biblionumber') || 0; my $frameworkcode = $input->param('frameworkcode'); @@ -73,11 +72,16 @@ $template->param( ); if ( $op ne "do_search" ) { - my $sth = $dbh->prepare("SELECT id,host,servername,checked FROM z3950servers WHERE recordtype='biblio' AND servertype='zed' ORDER BY rank, servername"); - $sth->execute(); - my $serverloop = $sth->fetchall_arrayref( {} ); + my $schema = Koha::Database->new()->schema(); + my $rs = $schema->resultset('Z3950server')->search( + { + recordtype => 'biblio', + servertype => ['zed', 'sru'], + }, + { result_class => 'DBIx::Class::ResultClass::HashRefInflator' }, + ); $template->param( - serverloop => $serverloop, + serverloop => [ $rs->all ], opsearch => "search", ); output_html_with_http_headers $input, $cookie, $template->output; -- 1.7.7.6