@@ -, +, @@ - Add a script in misc/maintenance/generate_explain.pl that can generate a new Explain file, based on Koha's config files and an optional extra config file. See the POD in the script for more details. - Replace the old Explain files provided with Koha with two new files, generated by the script described above and based on the standard Koha/Zebra config. - Have a look at etc/zebradb/explain-authorities.xml and etc/zebradb/explain-biblios.xml and see if they make sense as example files. Check that the list of indexes corresponds to the indexes set up in etc/zebradb/pqf.properties. - Run "perldoc misc/maintenance/generate_explain.pl" to see available options and a decription of the config file. - Run generate_explain.pl with different options and settings in the config file, and check that the output Explain docs make sense. --- etc/zebradb/explain-authorities.xml | 601 +++++++-------------- etc/zebradb/explain-biblios.xml | 591 +++++++------------- .../prog/en/modules/services/explain.tt | 260 +++++++++ misc/maintenance/generate_explain.pl | 331 ++++++++++++ 4 files changed, 976 insertions(+), 807 deletions(-) create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/services/explain.tt create mode 100644 misc/maintenance/generate_explain.pl --- a/etc/zebradb/explain-authorities.xml +++ a/etc/zebradb/explain-authorities.xml @@ -1,621 +1,417 @@ - - - - + - - - - - __ZEBRA_SRU_HOST__ - - 9999 - + + + library.example.org + + __ZEBRA_SRU_AUTHORITIES_PORT__ authorities - - - - Koha 3 Authority SRU/SRW/Z39.50 server - - - Koha 3 Authority Server - - - Joshua Ferraro, LibLime, http://liblime.com, - Marc Cromme, Mike Taylor, Index Data Aps, - http://www.indexdata.dk - - Joshua Ferraro, jmf(at)liblime(dot)com - - - - - - Created for Koha 3 - http://koha-community.org - - - + Koha Authorities SRU/SRW/Z39.50 server + Koha Authorities Server + My library + library@example.org + Index Data YAZ/ZEBRA http://www.indexdata.dk/zebra - - - - http://__ZEBRA_SRU_HOST__:__ZEBRA_SRU_AUTHORITIES_PORT__ - http://__ZEBRA_SRU_HOST__:__ZEBRA_SRU_AUTHORITIES_PORT__ - http://__ZEBRA_SRU_HOST__:__ZEBRA_SRU_AUTHORITIES_PORT__ - + + http://library.example.org/ + http://library.example.org:9999 + http://library.example.org:9999 + http://library.example.org:9999 + http://library.example.org/cgi-bin/koha/oai.pl - - - - - - - CQL Standard Set - - - Record Standard Set - - - Alvis Set - - - Dublin Core Set - - - BIB-1 Z39.50 Standard Set - - - - + + + + - CQL Server Choice - serverChoice - text + serverChoice - - - CQL All - all - - text + + 1016 - - - Record ID - id - rec:id - 3 + id - - - - - Alvis Text Relevance Ranked - text - - text - 102 + + 12 - Alvis Anchor Text - anchortext - anchortext - 102 + identifier - - - Alvis Date Modified - date-modified - - ac:date-modified - 5 + + 1007 - Alvis Date Checked - date-checked - ac:date-checked - 5 + title - - - Alvis Identifier Url - identifier-url - - ac:identifier-url - 3 + + 4 - - Alvis Topic Class - topic-class - an:topic-class + subject - - - Alvis Language - language - - an:language + + 21 - Alvis NLP Entity Date - entity-date - la:entity-date - 3 + creator + + + 1003 - Alvis NLP Entity Disease - entity-disease - la:entity-disease - 3 + author + + + 1003 - Alvis NLP Entity Gene - entity-gene - la:entity-gene - 3 + itemtype + + + 1031 - Alvis NLP Entity Location - entity-location - la:entity-location - 3 + barcode - - + - Alvis NLP Entity Person - entity-person - la:entity-person - 3 + branch - - + - Alvis NLP Entity Species - entity-species - la:entity-species - 3 + isbn + + + 7 - Alvis NLP Entity URL - entity-url - la:entity-url - 3 + issn + + + 8 - Alvis NLP Entity Organism - entity-organism - la:entity-organism - 3 + any + + + 1016 - Alvis NLP Entity Symptom - entity-symptom - la:entity-symptom - 3 + note + + + 63 - Alvis NLP Entity Chemical - entity-chemical - la:entity-chemical - 3 + pname + + + 1 - Alvis NLP Entity Drug - entity-drug - la:entity-drug - 3 + editor + + + 1020 - Alvis NLP Lemma Canonical - lemma-canonical - la:lemma-canonical - 3 + publisher + + + 1018 - Alvis NLP Syntactic Noun - syntactic-noun - la:syntactic-noun - 3 + description + + + 62 - Alvis NLP Syntactic Verb - syntactic-verb - la:syntactic-verb - 3 + date + + + 30 - Alvis NLP Syntactic Adjective - syntactic-adjective - la:syntactic-adjective - 3 + resourceType + + + 1031 - Alvis NLP Word Form - word-form - la:word-form - 3 + format + + + 1034 - - - - + - DC Title - title - dc:title + language + + + 54 - DC Creator - creator - dc:creator + Place-publication + + + 59 - DC Subject - subject - dc:subject + keyTitle + + + 33 - DC Description - description - dc:description + possessingInstitution + + + 1044 - DC Publisher - publisher - dc:publisher + name + + + 1002 - DC Contributor - contributor - dc:contributor + personalName + + + 1 - DC Date - date - dc:date + corporateName + + + 2 - DC Type - type - dc:type + conferenceName + + + 3 - DC Format - format - + uniformTitle + + + 6 - DC Identifier - identifier - dc:identifier + isbn + + + 7 - DC Source - source - dc:source + issn + + + 8 - DC Language - language - dc:language + geographicName + + + 58 - DC Relation - relation - dc:relation + notes + + + 63 - DC Coverage - coverage - dc:coverage + topicalSubject + + + 1079 - DC Rights - rights - dc:rights + genreForm + + + 1075 - - - - - - - - Dublin Core - - - - Dublin Core Short Form - - - - - - - - Pretty - - - - Alvis D9.1 Schema - - - - Alvis Enriched - - - - Zebra Indexing - - - - - - cql - all - = - - < - <= - = - >= - > - - - - + < + le + eq + exact + ge + > + <> + all + any + scr - - - - + relevant + fuzzy + stem + phonetic - - - + + - - * ? - - - + - - - - - - + 0 - index docpath/sru.xsl - xml - - - - + - - - - + 20 - - - 0 - - - - --- a/etc/zebradb/explain-biblios.xml +++ a/etc/zebradb/explain-biblios.xml @@ -1,621 +1,409 @@ - - - - + - - - - - __ZEBRA_SRU_HOST__ - + + + library.example.org 9999 - biblios - - - - Koha 3 Bibliographic SRU/SRW/Z39.50 server - - - Koha 3 Bibliographic Server - - - Joshua Ferraro, LibLime, http://liblime.com, - Marc Cromme, Mike Taylor, Index Data Aps, - http://www.indexdata.dk - - Joshua Ferraro, jmf(at)liblime(dot)com - - - - - - Created for Koha 3 - http://koha-community.org - - - + Koha Bibliographic SRU/SRW/Z39.50 server + Koha Bibliographic Server + My library + library@example.org + Index Data YAZ/ZEBRA http://www.indexdata.dk/zebra - - - - http://__ZEBRA_SRU_HOST__:__ZEBRA_SRU_BIBLIOS_PORT__ - http://__ZEBRA_SRU_HOST__:__ZEBRA_SRU_BIBLIOS_PORT__ - http://__ZEBRA_SRU_HOST__:__ZEBRA_SRU_BIBLIOS_PORT__ - + + http://library.example.org/ + http://library.example.org:9999 + http://library.example.org:9999 + http://library.example.org:9999 + http://library.example.org/cgi-bin/koha/oai.pl - - - - - - - CQL Standard Set - - - Record Standard Set - - - Alvis Set - - - Dublin Core Set - - - BIB-1 Z39.50 Standard Set - - - - + + + + - CQL Server Choice - serverChoice - text + serverChoice - - - CQL All - all - - text + + 1016 - - - Record ID - id - rec:id - 3 + id - - - - - Alvis Text Relevance Ranked - text - - text - 102 + + 12 - Alvis Anchor Text - anchortext - anchortext - 102 + identifier - - - Alvis Date Modified - date-modified - - ac:date-modified - 5 + + 1007 - Alvis Date Checked - date-checked - ac:date-checked - 5 + title - - - Alvis Identifier Url - identifier-url - - ac:identifier-url - 3 + + 4 - - Alvis Topic Class - topic-class - an:topic-class + subject - - - Alvis Language - language - - an:language + + 21 - Alvis NLP Entity Date - entity-date - la:entity-date - 3 + creator + + + 1003 - Alvis NLP Entity Disease - entity-disease - la:entity-disease - 3 + author + + + 1003 - Alvis NLP Entity Gene - entity-gene - la:entity-gene - 3 + itemtype + + + 1031 - Alvis NLP Entity Location - entity-location - la:entity-location - 3 + barcode - - + - Alvis NLP Entity Person - entity-person - la:entity-person - 3 + branch - - + - Alvis NLP Entity Species - entity-species - la:entity-species - 3 + isbn + + + 7 - Alvis NLP Entity URL - entity-url - la:entity-url - 3 + issn + + + 8 - Alvis NLP Entity Organism - entity-organism - la:entity-organism - 3 + any + + + 1016 - Alvis NLP Entity Symptom - entity-symptom - la:entity-symptom - 3 + note + + + 63 - Alvis NLP Entity Chemical - entity-chemical - la:entity-chemical - 3 + pname + + + 1 - Alvis NLP Entity Drug - entity-drug - la:entity-drug - 3 + editor + + + 1020 - Alvis NLP Lemma Canonical - lemma-canonical - la:lemma-canonical - 3 + publisher + + + 1018 - Alvis NLP Syntactic Noun - syntactic-noun - la:syntactic-noun - 3 + description + + + 62 - Alvis NLP Syntactic Verb - syntactic-verb - la:syntactic-verb - 3 + date + + + 30 - Alvis NLP Syntactic Adjective - syntactic-adjective - la:syntactic-adjective - 3 + resourceType + + + 1031 - Alvis NLP Word Form - word-form - la:word-form - 3 + format + + + 1034 - - - - + - DC Title - title - dc:title + language + + + 54 - DC Creator - creator - dc:creator + Place-publication + + + 59 - DC Subject - subject - dc:subject + keyTitle + + + 33 - DC Description - description - dc:description + possessingInstitution + + + 1044 - DC Publisher - publisher - dc:publisher + name + + + 1002 - DC Contributor - contributor - dc:contributor + personalName + + + 1 - DC Date - date - dc:date + corporateName + + + 2 - DC Type - type - dc:type + conferenceName + + + 3 - DC Format - format - + uniformTitle + + + 6 - DC Identifier - identifier - dc:identifier + isbn + + + 7 - DC Source - source - dc:source + issn + + + 8 - DC Language - language - dc:language + geographicName + + + 58 - DC Relation - relation - dc:relation + notes + + + 63 - DC Coverage - coverage - dc:coverage + topicalSubject + + + 1079 - DC Rights - rights - dc:rights + genreForm + + + 1075 - - - - - - - - Dublin Core - - - - Dublin Core Short Form - - - - - - - - Pretty - - - - Alvis D9.1 Schema - - - - Alvis Enriched - - - - Zebra Indexing - - - - - - cql - all - = - - < - <= - = - >= - > - - - - + < + le + eq + exact + ge + > + <> + all + any + scr - - - - + relevant + fuzzy + stem + phonetic - - - + + - - * ? - - - + - - - - - - + 0 - index docpath/sru.xsl - xml - - - - + - - - - + 20 - - - 0 - - - - --- a/koha-tmpl/intranet-tmpl/prog/en/modules/services/explain.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/services/explain.tt @@ -0,0 +1,260 @@ + + +[%- USE Koha -%] +[%- USE date( format = '%Y-%m-%d %H:%M:%S' ) -%] + + + + + + + + + + + + [% host %] + [% IF ( authorities ) %] + + __ZEBRA_SRU_AUTHORITIES_PORT__ + authorities + [% ELSE %] + [% port %] + biblios + [% END %] + + + + + [% IF ( config.title ) -%] + [%- FOREACH title IN config.title -%] + [% title.text %] + [% END -%] + [% ELSE -%] + [% IF ( authorities ) %] + Koha Authorities SRU/SRW/Z39.50 server + [% ELSE %] + Koha Bibliographic SRU/SRW/Z39.50 server + [% END %] + [% END -%] + + [%- IF ( config.description ) -%] + [%- FOREACH description IN config.description -%] + [% description.text %] + [% END -%] + [% ELSE -%] + [% IF ( authorities ) %] + Koha Authorities Server + [% ELSE %] + Koha Bibliographic Server + [% END %] + [% END -%] + + + [%- IF ( config.author ) -%] + [%- config.author -%] + [%- ELSIF ( Koha.Preference( 'LibraryName' ) != '' ) -%] + [% Koha.Preference( 'LibraryName' ) -%] + [%- ELSE -%] + Koha + [%- END -%] + + + [%- IF ( config.contact ) -%] + [%- config.contact -%] + [%- ELSIF ( Koha.Preference( 'KohaAdminEmailAddress' ) != '' ) -%] + [%- Koha.Preference( 'KohaAdminEmailAddress' ) -%] + [%- END -%] + + [% IF ( config.extent ) -%] + [%- FOREACH extent IN config.extent -%] + [% extent.text %] + [% END -%] + [% END -%] + + [% IF ( config.history ) -%] + [%- FOREACH history IN config.history -%] + [% history.text %] + [% END -%] + [% END -%] + + [% IF ( config.langUsage ) %] + [% FOREACH lang IN config.langUsage %] + [% lang.text %] + [% END %] + [% END %] + + [% IF ( config.restrictions ) -%] + [%- FOREACH restriction IN config.restrictions -%] + [% restriction.text %] + [% END -%] + [% END -%] + + [% IF ( config.subjects ) -%] + + [% FOREACH sub IN config.subjects -%] + [% sub %] + [% END -%] + + [% END %] + + + Index Data YAZ/ZEBRA http://www.indexdata.dk/zebra + + + + [% IF ( Koha.Preference( 'OPACBaseURL' ) ) %][% Koha.Preference( 'OPACBaseURL' ) %]/[% END %] + [% IF ( config.transport ) %][% config.transport %][% ELSE %]http[% END %]://[% host %]:[% port %] + [% IF ( config.transport ) %][% config.transport %][% ELSE %]http[% END %]://[% host %]:[% port %] + [% IF ( config.transport ) %][% config.transport %][% ELSE %]http[% END %]://[% host %]:[% port %] + [% IF ( Koha.Preference( 'OAI-PMH' ) ) %][% Koha.Preference( 'OPACBaseURL' ) %]/cgi-bin/koha/oai.pl[% END %] + [% IF ( Koha.Preference( 'OpacFavicon' ) ) %][% Koha.Preference( 'OpacFavicon' ) %][% END %] + + + + + + [% date.format %] + + + + + [% FOREACH set IN pqf.sets -%] + + [% END -%] + + [% FOREACH index IN pqf.indexes %] + + [%- IF config.indexnames.${index.id} -%] + [%- FOREACH this IN config.indexnames.${index.id} %] + [% this.title %] + [%- END %] + [%- END %] + [% index.index %] + + [% index.value %] + + + [% END -%] + + + + + + + + + cql + + all + + + = + + + [% FOREACH rel IN pqf.relations -%] + [% rel.rel | html %] + [% END %] + + + [% FOREACH relmod IN pqf.relationmodifiers -%] + [% relmod.relmod | html %] + [% END %] + + + + + + + + + + + + * + ? + + + + + + + + + + + + + + + + + + + + 0 + + + index + + docpath/sru.xsl + + + xml + + + + + + + + + + + + + + + + + 20 + + + + + + + + 0 + + + + + + + + + + + + + + --- a/misc/maintenance/generate_explain.pl +++ a/misc/maintenance/generate_explain.pl @@ -0,0 +1,331 @@ +#!/usr/bin/perl + +# Copyright 2017 Magnus Enger Libriotech +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +=head1 NAME + +generate_explain.pl + +=head1 SYNOPSIS + +sudo koha-shell -c "perl generate_explain.pl -v" mykoha + +sudo PERL5LIB=/usr/share/koha/lib KOHA_CONF=/etc/koha/sites/kohadev/koha-conf.xml perl generate_explain.pl -c /path/to/explain.yaml --overwrite -v + +=head1 DESCRIPTION + +Reads an optional config file, some of Koha's config files and generates an +Explain document that can be served by the Zebra SRU server. + +By default, the generated Explain document will be written to STDOUT. Specify +the --overwrite command line option to actually overwrite the existing Explain +document. + +=cut + +use File::Slurp; +use YAML::Syck qw( LoadFile ); +use CGI qw( utf8 ); # NOT a CGI script, this is just to keep C4::Templates::gettemplate happy +use XML::Simple; +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; +use Data::Dumper; # Debug + +use C4::Context; +use C4::Templates; + +=head1 OPTIONS + +=over 8 + +=item B<-c | --configfile> + +Path to a configfile that should be used to generate parts of the Explain +document. + +=item B<-a | --authorities> + +If this option is not specified, the script will assume we are dealing with +biblios. + +=item B<-o | --overwrite> + +Actually overwrite the existing explain document. If this is not specified, the +generated Explain document will be written to STDOUT and no files will be +overwritten. + +=item B<-h | --help> + +Print this usage statement. + +=item B<-v | --verbose> + +More verbose output. + +=back + +=cut + +my ( $configfile, $authorities, $overwrite, $help, $verbose ); +GetOptions( + 'c|configfile=s' => \$configfile, + 'a|authorities' => \$authorities, + 'o|overwrite' => \$overwrite, + 'h|help' => \$help, + 'v|verbose' => \$verbose, +); + +if ( $help ) { + pod2usage( -verbose => 2 ); + exit; +} + +=head1 CONFIG FILE + +The config file should be in YAML format. Here is an example config that shows +all the available options. See http://zeerex.z3950.org/dtd/commentary.html for +further details about the options. + + --- + # Protocol + # Default: SRW/SRU/Z39.50 + protocol: "SRU" + + # Transport + # Default: http + transport: "https" + + # Title + # Should have a two letter language code. + # One title can be marked as primary + title: + - text: This is some title + lang: en + primary: true + - text: Dette er en tittel + lang: nb + + # Description + # Should have a two letter language code. + # One description can be marked as primary + description: + - text: This is some description + lang: en + primary: true + - text: Dette er en beskrivelse + lang: nb + + # Author + # If author is not set in the config, the LibraryName syspref will be used. If + # that is not set, the string "Koha" will be used. + author: "The librarian" + + # Contact + # If the contact is not set in the config, the KohaAdminEmailAddress syspref + # will be used. + contact: "librarian@example.org" + + # Extent + extent: + - text: "This is the extent of the database." + lang: "en" + primary: "true" + + # History + history: + - text: "This is the history of the database." + lang: "en" + primary: "true" + + # langUsage + langUsage: + - text: "Records are in English and Norwegian" + codes: "en nb" + lang: "en" + primary: "true" + + # Restrictions + restrictions: + - text: "Free for all!" + lang: "en" + primary: "true" + + # Subjects + subjects: + - "Subject A" + - "Subject B" + + # Index names + # Indexes can be given more explanatory names, but this is optional. + indexnames: + dc_title: + - title: "Title" + lang: "en" + - title: "Tittel" + lang: "nb" + dc_subject: + - title: "Subject" + lang: "en" + rec_identifier: + - title: "Record identifier" + lang: "en" + +=cut + +# Load the config file, if one is specified +my $config; +if ( $configfile ) { + if ( -e $configfile ) { + say "Reading config from $configfile" if $verbose; + $config = LoadFile( $configfile ); + } else { + die "$configfile not found"; + } +} + +# Get the path to he Koha config file from the KOHA_CONF environment variable +my $koha_conf = $ENV{ 'KOHA_CONF' }; +say "Looking at $koha_conf" if $verbose; + +# Read the Koha config file +my $conf = XMLin( + $koha_conf, + keyattr => ['id'], + forcearray => ['listen', 'server', 'serverinfo'], + suppressempty => '' +); + +# Get the SRU host and port from the Koha config file +my $publicserver = $conf->{'listen'}->{'publicserver'}->{'content'}; +my ( $tcp, $host, $port ) = split /:/, $publicserver; +say "SRU is listening on host $host and port $port" if $verbose; + +# Find the path to pqf.properties file +my $pqf_properties_file = $conf->{'server'}->{'publicserver'}->{'cql2rpn'}; +say "Looking at $pqf_properties_file" if $verbose; + +# Read the pqf.properties file, and extract the information we need +my %pqf; +my @lines = read_file( $pqf_properties_file ); +foreach my $line ( @lines ) { + + # Remove trailing whitespace + chomp $line; + # Skip commented lines - start of line, zero or more spaces, comment sign + next if $line =~ m/^ {0,}#/; + # Skip empty lines - start of line, zero or more spaces, end of line + next if $line =~ m/^ {0,}$/; + + # Look for lines that start with "set." + if ( $line =~ m/^set\.(.*?) {0,}= {0,}(.*)/ ) { + push @{ $pqf{'sets'} }, { + 'name' => $1, + 'identifier' => $2, + }; + } + + # Look for lines that start with "index." + if ( $line =~ m/^index\.(.*?)\.(.*?) {0,}= (.*?)=(.*?) {0,}$/ ) { + push @{ $pqf{'indexes'} }, { + 'set' => $1, + 'index' => $2, + 'id' => "$1_$2", + 'type' => $3, + 'value' => $4, + }; + } + + # Look for lines that start with "relation." + if ( $line =~ m/^relation\.(.*?) {0,}=.*$/ ) { + push @{ $pqf{'relations'} }, { + 'rel' => $1, + }; + } + + # Look for lines that start with "relationModifier." + if ( $line =~ m/^relationModifier\.(.*?) {0,}=.*$/ ) { + push @{ $pqf{'relationmodifiers'} }, { + 'relmod' => $1, + }; + } + +} + +# Set up the template +my $template = C4::Templates::gettemplate( + 'services/explain.tt', + 'intranet', + new CGI +); + +# Pass some values to the template +$template->param( + 'config' => $config, + 'pqf' => \%pqf, + 'host' => $host, + 'port' => $port, + 'authorities' => $authorities, +); + +# Get the output from the template +my $output = $template->output; +if ( $overwrite ) { + # Find the path to the Explain doc + my $explain_doc; + foreach my $inc ( @{ $conf->{'server'}->{'publicserver'}->{'xi:include'} } ) { + my $href = $inc->{'href'}; + if ( $href =~ m/explain/ ) { + $explain_doc = $href; + } + } + say "Going to write to $explain_doc" if $verbose; + # Make a backup of the existing file + rename $explain_doc, "$explain_doc-old"; + # Do the actual write + write_file( $explain_doc, $output ); +} else { + say $output; +} + +=head1 TIPS AND TRICKS + +=head2 Pretty printing XML + +To have the output of this script look nice, run it through B: + + perl generate_explain.pl | xmllint --format - + +=head1 AUTHOR + +Magnus Enger, + +=head1 LICENSE + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +=cut --