Attachment 77787 Details for Bug 21210 – Bug 21210: Implement Elasticsearch based SRU server

[patch] Bug 21210: Implement Elasticsearch based SRU server

0001-Bug-21210-Implement-Elasticsearch-based-SRU-server.patch (text/plain), 13.34 KB, created by Joonas Kylmälä on 2018-08-14 13:49:04 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Joonas Kylmälä

Created: 2018-08-14 13:49:04 UTC

Size: 13.34 KB

patch

obsolete

>From 74703ade92043c2cc137260ad60d954f1e88ad23 Mon Sep 17 00:00:00 2001
>From: =?UTF-8?q?Joonas=20Kylm=C3=A4l=C3=A4?= <joonas.kylmala@helsinki.fi>
>Date: Tue, 14 Aug 2018 15:59:34 +0300
>Subject: [PATCH] Bug 21210: Implement Elasticsearch based SRU server
>
>To test:
>
>1. Make sure you have specified a working elasticsearch configuration in
>   koha-conf.xml and biblios index is populated with some data
>
>2. Start SRU server by running "koha-dev/bin/koha-sru-server.sh start"
>
>3. Try it out, for example: $ curl
>   "http://localhost:9998/sru?version=1.1&operation=searchRetrieve&query=dc.title=Test".
>   if you get back a bibliographic record all is good!
>---
> Makefile.PL                 |   1 +
> etc/sru_server/catmandu.yml | 208 ++++++++++++++++++++++++++++++++++++++++++++
> etc/sru_server/config.yml   |  15 ++++
> etc/sru_server/marcxml.tt   |  19 ++++
> misc/bin/koha-sru-server.sh |  90 +++++++++++++++++++
> misc/sru_server.pl          |  39 +++++++++
> 6 files changed, 372 insertions(+)
> create mode 100644 etc/sru_server/catmandu.yml
> create mode 100644 etc/sru_server/config.yml
> create mode 100644 etc/sru_server/marcxml.tt
> create mode 100755 misc/bin/koha-sru-server.sh
> create mode 100755 misc/sru_server.pl
>
>diff --git a/Makefile.PL b/Makefile.PL
>index 74f74f6..7172d40 100644
>--- a/Makefile.PL
>+++ b/Makefile.PL
>@@ -630,6 +630,7 @@ push @{ $pl_files->{'rewrite-config.PL'} }, (
>     'blib/SCRIPT_DIR/koha-zebra-ctl.sh',
>     'blib/SCRIPT_DIR/koha-pazpar2-ctl.sh',
>     'blib/SCRIPT_DIR/koha-index-daemon-ctl.sh',
>+    'blib/SCRIPT_DIR/koha-sru-server.sh',
> );
> if ($config{'INSTALL_PAZPAR2'} eq 'yes') {
>     push @{ $pl_files->{'rewrite-config.PL'} }, (
>diff --git a/etc/sru_server/catmandu.yml b/etc/sru_server/catmandu.yml
>new file mode 100644
>index 0000000..c57be73
>--- /dev/null
>+++ b/etc/sru_server/catmandu.yml
>@@ -0,0 +1,208 @@
>+---
>+store:
>+  sru:
>+    package: ElasticSearch
>+    options:
>+      bags:
>+        data:
>+          cql_mapping:
>+            # Make sure to keep all index names lowercase
>+            default_index: cql.serverchoice
>+            indexes:
>+              rec.id:
>+                op:
>+                  '=': true
>+                  'exact': true
>+                field: 'Local-number'
>+              dc.identifier:
>+                op:
>+                  '=': true
>+                  'exact': true
>+                field: 'control-number'
>+              dc.title:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'title'
>+              dc.subject:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'subject'
>+              dc.creator:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'author'
>+              dc.author:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'author'
>+              dc.itemtype:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'itype'
>+              dc.branch:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: ['homebranch', 'holdingbranch']
>+              dc.isbn:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'isbn'
>+              dc.issn:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'issn'
>+              dc.any:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: '_all'
>+              dc.pname:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'Personal-name'
>+              dc.publisher:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'publisher'
>+              dc.date:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'copydate'
>+              dc.format:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'material-type'
>+              dc.resourceidentifier:
>+                op:
>+                  '=': true
>+                  'exact': true
>+                field: 'Local-number'
>+              bath.keytitle:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'title'
>+              bath.possessinginstitution:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: ['homebranch', 'holdingbranch']
>+              bath.name:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'author'
>+              bath.personalname:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'Personal-name'
>+              bath.corporatename:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'author'
>+              bath.conferencename:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'Meeting-name'
>+              bath.isbn:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'isbn'
>+              bath.issn:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'issn'
>+              bath.geographicname:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'su-geo'
>+              bath.topicalsubject:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'subject'
>+              cql.serverchoice:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: '_all'
>+              koha.titlemain:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'title-main'
>+              koha.biblevel:
>+                op:
>+                  'any': true
>+                  'all': true
>+                  '=': true
>+                  'exact': true
>+                field: 'bib-level'
>diff --git a/etc/sru_server/config.yml b/etc/sru_server/config.yml
>new file mode 100644
>index 0000000..1eb55de
>--- /dev/null
>+++ b/etc/sru_server/config.yml
>@@ -0,0 +1,15 @@
>+charset: "UTF-8"
>+content_type: "text/xml"
>+logger: console
>+plugins:
>+    'Catmandu::SRU':
>+        store: sru
>+        bag: data
>+        default_record_schema: marcxml
>+        limit: 20
>+        maximum_limit: 1000
>+        record_schemas:
>+            -
>+                identifier: "info:srw/schema/1/marcxml-v1.1"
>+                name: marcxml
>+                template: marcxml.tt
>diff --git a/etc/sru_server/marcxml.tt b/etc/sru_server/marcxml.tt
>new file mode 100644
>index 0000000..1ab9bf4
>--- /dev/null
>+++ b/etc/sru_server/marcxml.tt
>@@ -0,0 +1,19 @@
>+
>+<record xmlns="http://www.loc.gov/MARC21/slim">
>+[%- FOREACH field IN record %]
>+  [%- IF field.0 == 'LDR' %]
>+  <leader>[% field.4 | xml %]</leader>
>+  [%- ELSIF field.0 < 10 %]
>+  <controlfield tag="[% field.0 | xml %]">[% field.4 | xml %]</controlfield>
>+  [%- ELSE %]
>+  <datafield tag="[% field.0 | xml %]" ind1="[% field.1 | xml %]" ind2="[% field.2 | xml %]">
>+    [%- c = 3 %]
>+    [%- WHILE c < field.size %]
>+      [%- di = c + 1 %]
>+    <subfield code="[% field.$c | xml %]">[% field.$di | xml %]</subfield>
>+      [%- c = c + 2 %]
>+    [%- END %]
>+    </datafield>
>+  [%- END %]
>+[%- END %]
>+</record>
>diff --git a/misc/bin/koha-sru-server.sh b/misc/bin/koha-sru-server.sh
>new file mode 100755
>index 0000000..3f74a82
>--- /dev/null
>+++ b/misc/bin/koha-sru-server.sh
>@@ -0,0 +1,90 @@
>+#!/bin/sh
>+
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it
>+# under the terms of the GNU General Public License as published by
>+# the Free Software Foundation; either version 3 of the License, or
>+# (at your option) any later version.
>+#
>+# Koha is distributed in the hope that it will be useful, but
>+# WITHOUT ANY WARRANTY; without even the implied warranty of
>+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
>+# GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License
>+# along with Koha; if not, see <http://www.gnu.org/licenses>.
>+
>+### BEGIN INIT INFO
>+# Provides:          koha-sru-server-$DBNAME
>+# Required-Start:    $local_fs $syslog
>+# Required-Stop:     $local_fs $syslog
>+# Default-Start:     2 3 4 5
>+# Default-Stop:      0 1 6
>+# X-Interactive:     false
>+# Short-Description: Start/stop koha-sru-server for $DBNAME
>+### END INIT INFO
>+
>+. /lib/lsb/init-functions
>+
>+USER=__KOHA_USER__
>+GROUP=__KOHA_GROUP__
>+DBNAME=__DB_NAME__
>+NAME=koha-sru-server-$DBNAME
>+PORT=9998
>+LOGDIR=__LOG_DIR__
>+PERL5LIB=__PERL_MODULE_DIR__
>+KOHA_CONFDIR=__KOHA_CONF_DIR__
>+KOHA_CONF=$KOHA_CONFDIR/koha-conf.xml
>+SRU_CONFDIR=$KOHA_CONFDIR/sru_server
>+DANCER_CONFDIR=$SRU_CONFDIR
>+ERRLOG=$LOGDIR/koha-sru-server.err
>+STDOUT=$LOGDIR/koha-sru-server.log
>+OUTPUT=$LOGDIR/koha-sru-server-output.log
>+
>+export KOHA_CONF
>+export PERL5LIB
>+export DANCER_CONFDIR
>+
>+SRUSERVER="/usr/bin/plackup --server Starman --port $PORT $PERL5LIB/misc/sru_server.pl"
>+SRUSERVER_OPTS=""
>+
>+DAEMONOPTS="--name=$NAME \
>+            --errlog=$ERRLOG \
>+            --stdout=$STDOUT \
>+            --output=$OUTPUT \
>+            --verbose=1 --respawn --delay=30"
>+
>+USER="--user=$USER.$GROUP"
>+
>+
>+case "$1" in
>+    start)
>+      log_daemon_msg "Starting Koha SRU server ($DBNAME)"
>+      if daemon $DAEMONOPTS $USER -- $SRUSERVER $SRUSERVER_OPTS; then
>+        log_end_msg 0
>+      else
>+        log_end_msg 1
>+      fi
>+      ;;
>+    stop)
>+      log_daemon_msg "Stopping Koha SRU server ($DBNAME)"
>+      if daemon $DAEMONOPTS $USER --stop -- $SRUSERVER $SRUSERVER_OPTS; then
>+        log_end_msg 0
>+      else
>+        log_end_msg 1
>+      fi
>+      ;;
>+    restart)
>+      log_daemon_msg "Restarting the Koha SRU server ($DBNAME)"
>+      if daemon $DAEMONOPTS $USER --restart -- $SRUSERVER $SRUSERVER_OPTS; then
>+        log_end_msg 0
>+      else
>+        log_end_msg 1
>+      fi
>+      ;;
>+    *)
>+      log_success_msg "Usage: /etc/init.d/$NAME {start|stop|restart}"
>+      exit 1
>+      ;;
>+esac
>diff --git a/misc/sru_server.pl b/misc/sru_server.pl
>new file mode 100755
>index 0000000..10f726a
>--- /dev/null
>+++ b/misc/sru_server.pl
>@@ -0,0 +1,39 @@
>+#!/usr/bin/env perl
>+#
>+# Copyright University of Helsinki 2017
>+#
>+# This file is part of Koha.
>+#
>+# Koha is free software; you can redistribute it and/or modify it under the
>+# terms of the GNU General Public License as published by the Free Software
>+# Foundation; either version 3 of the License, or (at your option) any later
>+# version.
>+#
>+# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
>+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
>+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
>+#
>+# You should have received a copy of the GNU General Public License along
>+# with Koha; if not, write to the Free Software Foundation, Inc.,
>+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
>+
>+use Dancer;
>+use Dancer::Config;
>+use Catmandu;
>+use Dancer::Plugin::Catmandu::SRU;
>+use Koha::Logger;
>+use Koha::SearchEngine::Elasticsearch;
>+
>+Koha::Logger->get;
>+
>+Catmandu->load( Dancer::Config::setting('confdir') );
>+
>+my $catmanduConfig = Catmandu->config;
>+my $es = Koha::SearchEngine::Elasticsearch->new({index => 'biblios'});
>+
>+$catmanduConfig->{store}{sru}{options}{nodes} = $es->get_elasticsearch_params()->{nodes};
>+$catmanduConfig->{store}{sru}{options}{index_name} = $es->get_elasticsearch_params()->{index_name};
>+
>+sru_provider '/sru';
>+
>+dance;
>-- 
>2.7.4
>

Actions: View | Diff | Splinter Review

Attachments on bug 21210: 77787