Attachment 30935 Details for Bug 6536 – [PASSED QA] Bug 6536: Include SRU searching in Breeding.pm

[patch] [PASSED QA] Bug 6536: Include SRU searching in Breeding.pm

PASSED-QA-Bug-6536-Include-SRU-searching-in-Breedi.patch (text/plain), 14.58 KB, created by Martin Renvoize (ashimema) on 2014-08-18 13:48:21 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Martin Renvoize (ashimema)

Created: 2014-08-18 13:48:21 UTC

Size: 14.58 KB

patch

obsolete

>From b0f954b59c5646475cfc0800d81753a3439fcb48 Mon Sep 17 00:00:00 2001
>From: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
>Date: Thu, 5 Jun 2014 14:20:37 +0200
>Subject: [PATCH] [PASSED QA] Bug 6536: Include SRU searching in Breeding.pm
>
>This patch makes it possible to include SRU servers in Z3950 search.
>It adjusts the Z3950Search routine in Breeding module.
>It also replaces SQL code with DBIx statements in Breeding.pm/Z3950Search
>and the associated scripts z3950search.pl in cataloguing and acqui.
>
>Test plan:
>Verify if a normal Z3950 search still works in cataloging/acqui.
>Add a SRU target. (You could just use Koha's port 9998.)
>Define sru_options like sru=get.
>Use that target in a Z3950 search in cataloging and acqui. (Import.)
>Test sru_fields translation by comparing search results between various
>settings for some of the fields. For instance, leave title empty and
>after that set it to the title field of your SRU target.
>
>Signed-off-by: Giuseppe Angilella <giuseppe.angilella@ct.infn.it>
>Signed-off-by: Martin Renvoize <martin.renvoize@ptfs-europe.com>
>---
> C4/Breeding.pm              |  222 +++++++++++++++++++++++--------------------
> acqui/z3950_search.pl       |   14 ++-
> cataloguing/z3950_search.pl |   14 ++-
> 3 files changed, 139 insertions(+), 111 deletions(-)
>
>diff --git a/C4/Breeding.pm b/C4/Breeding.pm
>index 0534665..2001fa5 100644
>--- a/C4/Breeding.pm
>+++ b/C4/Breeding.pm
>@@ -27,6 +27,7 @@ use C4::Charset;
> use MARC::File::USMARC;
> use C4::ImportBatch;
> use C4::AuthoritiesMarc; #GuessAuthTypeCode, FindDuplicateAuthority
>+use Koha::Database;
> 
> use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
> 
>@@ -135,109 +136,36 @@ sub Z3950Search {
>     my @id= @{$pars->{id}};
>     my $page= $pars->{page};
>     my $biblionumber= $pars->{biblionumber};
>-    my $isbn= $pars->{isbn};
>-    my $issn= $pars->{issn};
>-    my $title= $pars->{title};
>-    my $author= $pars->{author};
>-    my $dewey= $pars->{dewey};
>-    my $subject= $pars->{subject};
>-    my $lccn= $pars->{lccn};
>-    my $lccall= $pars->{lccall};
>-    my $controlnumber= $pars->{controlnumber};
>-    my $srchany= $pars->{srchany};
>-    my $stdid= $pars->{stdid};
> 
>     my $show_next       = 0;
>     my $total_pages     = 0;
>-    my $term;
>     my @results;
>     my @breeding_loop = ();
>     my @oConnection;
>     my @oResult;
>     my @errconn;
>     my $s = 0;
>-    my $query;
>-    my $nterms=0;
>     my $imported=0;
>-    my @serverinfo; #replaces former serverhost, servername, encoding
> 
>-    if ($isbn) {
>-        $term=$isbn;
>-        $query .= " \@attr 1=7 \@attr 5=1 \"$term\" ";
>-        $nterms++;
>-    }
>-    if ($issn) {
>-        $term=$issn;
>-        $query .= " \@attr 1=8 \@attr 5=1 \"$term\" ";
>-        $nterms++;
>-    }
>-    if ($title) {
>-        $query .= " \@attr 1=4 \"$title\" ";
>-        $nterms++;
>-    }
>-    if ($author) {
>-        $query .= " \@attr 1=1003 \"$author\" ";
>-        $nterms++;
>-    }
>-    if ($dewey) {
>-        $query .= " \@attr 1=16 \"$dewey\" ";
>-        $nterms++;
>-    }
>-    if ($subject) {
>-        $query .= " \@attr 1=21 \"$subject\" ";
>-        $nterms++;
>-    }
>-    if ($lccn) {
>-        $query .= " \@attr 1=9 $lccn ";
>-        $nterms++;
>-    }
>-    if ($lccall) {
>-        $query .= " \@attr 1=16 \@attr 2=3 \@attr 3=1 \@attr 4=1 \@attr 5=1 \@attr 6=1 \"$lccall\" ";
>-        $nterms++;
>-    }
>-    if ($controlnumber) {
>-        $query .= " \@attr 1=12 \"$controlnumber\" ";
>-        $nterms++;
>-    }
>-    if($srchany) {
>-        $query .= " \@attr 1=1016 \"$srchany\" ";
>-        $nterms++;
>-    }
>-    if($stdid) {
>-        $query .= " \@attr 1=1007 \"$stdid\" ";
>-        $nterms++;
>-    }
>-    for my $i (1..$nterms-1) {
>-        $query = "\@and " . $query;
>-    }
>-
>-    my $dbh   = C4::Context->dbh;
>-    foreach my $servid (@id) {
>-        my $sth = $dbh->prepare("select * from z3950servers where id=?");
>-        $sth->execute($servid);
>-        while (my $server = $sth->fetchrow_hashref) {
>-            my $option1= new ZOOM::Options();
>-            $option1->option( 'async' => 1 );
>-            $option1->option( 'elementSetName', 'F' );
>-            $option1->option( 'databaseName',   $server->{db} );
>-            $option1->option( 'user', $server->{userid} ) if $server->{userid};
>-            $option1->option( 'password', $server->{password} ) if $server->{password};
>-            $option1->option( 'preferredRecordSyntax', $server->{syntax} );
>-            $option1->option( 'timeout', $server->{timeout} ) if $server->{timeout};
>-            $oConnection[$s]= create ZOOM::Connection($option1);
>-            $oConnection[$s]->connect( $server->{host}, $server->{port} );
>-            $serverinfo[$s]->{host}= $server->{host};
>-            $serverinfo[$s]->{name}= $server->{servername};
>-            $serverinfo[$s]->{encd}= $server->{encoding} // "iso-5426";
>-            $s++;
>-        }    ## while fetch
>-    }    # foreach
>-    my $nremaining  = $s;
>-
>-    for ( my $z = 0 ; $z < $s ; $z++ ) {
>-        $oResult[$z] = $oConnection[$z]->search_pqf($query);
>-    }
>+    my ( $zquery, $squery ) = _build_query( $pars );
> 
>+    my $schema = Koha::Database->new()->schema();
>+    my $rs = $schema->resultset('Z3950server')->search(
>+        { id => [ @id ] },
>+        { result_class => 'DBIx::Class::ResultClass::HashRefInflator' },
>+    );
>+    my @servers = $rs->all;
>+    foreach my $server ( @servers ) {
>+        $oConnection[$s] = _create_connection( $server );
>+        $oResult[$s] =
>+            $server->{servertype} eq 'zed'?
>+                $oConnection[$s]->search_pqf( $zquery ):
>+                $oConnection[$s]->search(new ZOOM::Query::CQL(
>+                    _translate_query( $server, $squery )));
>+        $s++;
>+    }
>+
>+    my $nremaining = $s;
>     while ( $nremaining-- ) {
>         my $k;
>         my $event;
>@@ -251,7 +179,7 @@ sub Z3950Search {
>             my ($error)= $oConnection[$k]->error_x(); #ignores errmsg, addinfo, diagset
>             if ($error) {
>                 if ($error =~ m/^(10000|10007)$/ ) {
>-                    push(@errconn, { server => $serverinfo[$k]->{host}, error => $error } );
>+                    push(@errconn, { server => $servers[$k]->{host}, error => $error } );
>                 }
>             }
>             else {
>@@ -263,11 +191,11 @@ sub Z3950Search {
>                     $total_pages = int($numresults/20)+1 if $total_pages < ($numresults/20);
>                     for ($i = ($page-1)*20; $i < (($numresults < ($page*20)) ? $numresults : ($page*20)); $i++) {
>                         if($oResult[$k]->record($i)) {
>-                            my $res=_handle_one_result($oResult[$k]->record($i), $serverinfo[$k], ++$imported, $biblionumber); #ignores error in sequence numbering
>+                            my $res=_handle_one_result($oResult[$k]->record($i), $servers[$k], ++$imported, $biblionumber); #ignores error in sequence numbering
>                             push @breeding_loop, $res if $res;
>                         }
>                         else {
>-                            push(@breeding_loop,{'server'=>$serverinfo[$k]->{name},'title'=>join(': ',$oConnection[$k]->error_x()),'breedingid'=>-1,'biblionumber'=>-1});
>+                            push(@breeding_loop,{'server'=>$servers[$k]->{servername},'title'=>join(': ',$oConnection[$k]->error_x()),'breedingid'=>-1,'biblionumber'=>-1});
>                         }
>                     }
>                 }    #if $numresults
>@@ -289,10 +217,6 @@ sub Z3950Search {
>         $oConnection[$_]->destroy();
>     }
> 
>-    my @servers = ();
>-    foreach my $id (@id) {
>-        push @servers, {id => $id};
>-    }
>     $template->param(
>         breeding_loop => \@breeding_loop,
>         servers => \@servers,
>@@ -300,14 +224,53 @@ sub Z3950Search {
>     );
> }
> 
>+sub _build_query {
>+    my ( $pars ) = @_;
>+
>+    my $qry_build = {
>+        isbn    => '@attr 1=7 @attr 5=1 "#term" ',
>+        issn    => '@attr 1=8 @attr 5=1 "#term" ',
>+        title   => '@attr 1=4 #term ',
>+        author  => '@attr 1=1003 "#term" ',
>+        dewey   => '@attr 1=16 "#term" ',
>+        subject => '@attr 1=21 "#term" ',
>+        lccall  => '@attr 1=16 @attr 2=3 @attr 3=1 @attr 4=1 @attr 5=1 '.
>+                   '@attr 6=1 "#term" ',
>+        controlnumber => '@attr 1=12 "#term" ',
>+        srchany => '@attr 1=1016 "#term" ',
>+        stdid   => '@attr 1=1007 "#term" ',
>+    };
>+
>+    my $zquery='';
>+    my $squery='';
>+    my $nterms=0;
>+    foreach my $k ( keys %$pars ) {
>+        if( ( my $val=$pars->{$k} ) && $qry_build->{$k} ) {
>+            $qry_build->{$k} =~ s/#term/$val/g;
>+            $zquery .= $qry_build->{$k};
>+            $squery .= "[$k]=\"$val\" and ";
>+            $nterms++;
>+        }
>+    }
>+    $zquery = "\@and " . $zquery for 2..$nterms;
>+    $squery =~ s/ and $//;
>+    return ( $zquery, $squery );
>+}
>+
> sub _handle_one_result {
>     my ($zoomrec, $servhref, $seq, $bib)= @_;
> 
>     my $raw= $zoomrec->raw();
>-    my ($marcrecord) = MarcToUTF8Record($raw, C4::Context->preference('marcflavour'), $servhref->{encd}); #ignores charset return values
>+    my $marcrecord;
>+    if( $servhref->{servertype} eq 'sru' ) {
>+        $marcrecord= MARC::Record->new_from_xml( $raw, 'UTF-8',
>+            $servhref->{syntax} );
>+    } else {
>+        ($marcrecord) = MarcToUTF8Record($raw, C4::Context->preference('marcflavour'), $servhref->{encoding} // "iso-5426" ); #ignores charset return values
>+    }
>     SetUTF8Flag($marcrecord);
> 
>-    my $batch_id = GetZ3950BatchId($servhref->{name});
>+    my $batch_id = GetZ3950BatchId($servhref->{servername});
>     my $breedingid = AddBiblioToBatch($batch_id, $seq, $marcrecord, 'UTF-8', 0, 0);
>         #FIXME passing 0 for z3950random
>         #Will eliminate this unused field in a followup report
>@@ -319,7 +282,7 @@ sub _handle_one_result {
>     return _add_rowdata(
>         {
>             biblionumber => $bib,
>-            server       => $servhref->{name},
>+            server       => $servhref->{servername},
>             breedingid   => $breedingid,
>         }, $marcrecord) if $breedingid;
> }
>@@ -354,6 +317,63 @@ sub _isbn_replace {
>     return $isbn;
> }
> 
>+sub _create_connection {
>+    my ( $server ) = @_;
>+    my $option1= new ZOOM::Options();
>+    $option1->option( 'async' => 1 );
>+    $option1->option( 'elementSetName', 'F' );
>+    $option1->option( 'preferredRecordSyntax', $server->{syntax} );
>+    $option1->option( 'timeout', $server->{timeout} ) if $server->{timeout};
>+
>+    if( $server->{servertype} eq 'sru' ) {
>+        foreach( split ',', $server->{sru_options}//'' ) {
>+            my @temp= split '=';
>+            $option1->option( $temp[0] => $temp[1] ) if @temp;
>+        }
>+    } elsif( $server->{servertype} eq 'zed' ) {
>+        $option1->option( 'databaseName',   $server->{db} );
>+        $option1->option( 'user', $server->{userid} ) if $server->{userid};
>+        $option1->option( 'password', $server->{password} ) if $server->{password};
>+    }
>+
>+    my $obj= ZOOM::Connection->create($option1);
>+    if( $server->{servertype} eq 'sru' ) {
>+        my $host= $server->{host};
>+        if( $host !~ /^https?:\/\// ) {
>+            #Normally, host will not be prefixed by protocol.
>+            #In that case we can (safely) assume http.
>+            #In case someone prefixed with https, give it a try..
>+            $host = 'http://' . $host;
>+        }
>+        $obj->connect( $host.':'.$server->{port}.'/'.$server->{db} );
>+    } else {
>+        $obj->connect( $server->{host}, $server->{port} );
>+    }
>+    return $obj;
>+}
>+
>+sub _translate_query { #SRU query adjusted per server cf. srufields column
>+    my ($server, $query) = @_;
>+
>+    #sru_fields is in format title=field,isbn=field,...
>+    #if a field doesn't exist, try anywhere or remove [field]=
>+    my @parts= split(',', $server->{sru_fields} );
>+    my %trans= map { if( /=/ ) { ( $`,$' ) } else { () } } @parts;
>+    my $any= $trans{srchany}?$trans{srchany}.'=':'';
>+
>+    my $q=$query;
>+    foreach my $key (keys %trans) {
>+        my $f=$trans{$key};
>+        if( $f ) {
>+            $q=~s/\[$key\]/$f/g;
>+        } else {
>+            $q=~s/\[$key\]=/$any/g;
>+        }
>+    }
>+    $q=~s/\[\w+\]=/$any/g; # remove remaining fields (not found in field list)
>+    return $q;
>+}
>+
> =head2 ImportBreedingAuth
> 
> ImportBreedingAuth($marcrecords,$overwrite_auth,$filename,$encoding,$z3950random,$batch_type);
>diff --git a/acqui/z3950_search.pl b/acqui/z3950_search.pl
>index 6032b12..f87354f 100755
>--- a/acqui/z3950_search.pl
>+++ b/acqui/z3950_search.pl
>@@ -31,7 +31,6 @@ use C4::Koha;
> use C4::Bookseller qw/ GetBookSellerFromId /;
> 
> my $input           = new CGI;
>-my $dbh             = C4::Context->dbh;
> my $biblionumber    = $input->param('biblionumber')||0;
> my $frameworkcode   = $input->param('frameworkcode')||'';
> my $title           = $input->param('title');
>@@ -92,11 +91,16 @@ $template->param(
> );
> 
> if ( $op ne "do_search" ) {
>-    my $sth = $dbh->prepare("select id,host,servername,checked from z3950servers where recordtype='biblio' and servertype='zed' order by host");
>-    $sth->execute();
>-    my $serverloop = $sth->fetchall_arrayref( {} );
>+    my $schema = Koha::Database->new()->schema();
>+    my $rs = $schema->resultset('Z3950server')->search(
>+        {
>+            recordtype => 'biblio',
>+            servertype => ['zed', 'sru'],
>+        },
>+        { result_class => 'DBIx::Class::ResultClass::HashRefInflator' },
>+    );
>     $template->param(
>-        serverloop   => $serverloop,
>+        serverloop   => [ $rs->all ],
>         opsearch     => "search",
>     );
>     output_html_with_http_headers $input, $cookie, $template->output;
>diff --git a/cataloguing/z3950_search.pl b/cataloguing/z3950_search.pl
>index a4afdcc..22dc565 100755
>--- a/cataloguing/z3950_search.pl
>+++ b/cataloguing/z3950_search.pl
>@@ -29,7 +29,6 @@ use C4::Breeding;
> use C4::Koha;
> 
> my $input        = new CGI;
>-my $dbh          = C4::Context->dbh;
> my $error         = $input->param('error');
> my $biblionumber  = $input->param('biblionumber') || 0;
> my $frameworkcode = $input->param('frameworkcode');
>@@ -73,11 +72,16 @@ $template->param(
> );
> 
> if ( $op ne "do_search" ) {
>-    my $sth = $dbh->prepare("SELECT id,host,servername,checked FROM z3950servers WHERE recordtype='biblio' AND servertype='zed' ORDER BY rank, servername");
>-    $sth->execute();
>-    my $serverloop = $sth->fetchall_arrayref( {} );
>+    my $schema = Koha::Database->new()->schema();
>+    my $rs = $schema->resultset('Z3950server')->search(
>+        {
>+            recordtype => 'biblio',
>+            servertype => ['zed', 'sru'],
>+        },
>+        { result_class => 'DBIx::Class::ResultClass::HashRefInflator' },
>+    );
>     $template->param(
>-        serverloop   => $serverloop,
>+        serverloop   => [ $rs->all ],
>         opsearch     => "search",
>     );
>     output_html_with_http_headers $input, $cookie, $template->output;
>-- 
>1.7.10.4

Actions: View | Diff | Splinter Review

Attachments on bug 6536: 4540 | 4541 | 4959 | 6901 | 6919 | 6920 | 6924 | 13462 | 26169 | 26170 | 26171 | 26172 | 29754 | 29755 | 29756 | 29779 | 29780 | 29788 | 29790 | 29791 | 29812 | 29813 | 29814 | 29815 | 29816 | 29817 | 29818 | 29819 | 30189 | 30190 | 30191 | 30192 | 30193 | 30194 | 30900 | 30901 | 30902 | 30903 | 30904 | 30905 | 30906 | 30907 | 30929 | 30930 | 30931 | 30932 | 30933 | 30934 | 30935 | 30936 | 31046 | 31048 | 31137 | 31193 | 31194 | 31195 | 31227 | 31247 | 31248 | 37264