From e1133c872dee6b0754785c983e0efa5c32081631 Mon Sep 17 00:00:00 2001 From: Julian Maurice <julian.maurice@biblibre.com> Date: Mon, 23 Jan 2012 16:25:51 +0100 Subject: [PATCH] Bug 6440: [SIGNED-OFF] Implement OAI-PMH Sets New sql tables: - oai_sets: contains the list of sets, described by a spec and a name - oai_sets_descriptions: contains a list of descriptions for each set - oai_sets_mappings: conditions on marc fields to match for biblio to be in a set - oai_sets_biblios: list of biblionumbers for each set New admin page: allow to configure sets: - Creation, deletion, modification of spec, name and descriptions - Define mappings which will be used for building oai sets Implements OAI Sets in opac/oai.pl: - ListSets, ListIdentifiers, ListRecords, GetRecord New script misc/migration_tools/build_oai_sets.pl: - Retrieve marcxml from all biblios and test if they belong to defined sets. The oai_sets_biblios table is then updated accordingly New system preference OAI-PMH:AutoUpdateSets. If on, update sets automatically when a biblio is created or updated. Use OPACBaseURL in oai_dc xslt Signed-off-by: Magnus Enger <magnus@enger.priv.no> I created a set based on a value in 245$a, ran the script and verified that the created st contained the expected records. I repeated this with a value from 952$a, to test item level data. Both were OK. Then i set OAI-PMH:AutoUpdateSets = Enable and created a set based on a value in 245$a, built the set with the script and verified that it looked OK. Then i changed the value in 245$a and verified that the set was updated without running the script. I repeated this with a value for 952$a. This did not work initially, but this has been fixed by the followup patch. All the options to build_oai_sets.pl work as expected. See comments on the bug for things that have been fixed along the way. --- C4/Biblio.pm | 12 + C4/OAI/Sets.pm | 589 ++++++++++++++++++++ admin/oai_set_mappings.pl | 86 +++ admin/oai_sets.pl | 102 ++++ installer/data/mysql/atomicupdate/oai_sets.sql | 35 ++ installer/data/mysql/kohastructure.sql | 49 ++ installer/data/mysql/sysprefs.sql | 1 + installer/data/mysql/updatedatabase.pl | 10 + .../intranet-tmpl/prog/en/includes/admin-menu.inc | 1 + .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/oai_set_mappings.tt | 103 ++++ .../prog/en/modules/admin/oai_sets.tt | 140 +++++ .../en/modules/admin/preferences/web_services.pref | 6 + .../prog/en/modules/help/admin/oai_set_mappings.tt | 40 ++ .../prog/en/modules/help/admin/oai_sets.tt | 49 ++ .../prog/en/xslt/UNIMARCslim2OAIDC.xsl | 11 +- misc/migration_tools/build_oai_sets.pl | 165 ++++++ opac/oai.pl | 233 ++++++-- 18 files changed, 1578 insertions(+), 56 deletions(-) create mode 100644 C4/OAI/Sets.pm create mode 100755 admin/oai_set_mappings.pl create mode 100755 admin/oai_sets.pl create mode 100644 installer/data/mysql/atomicupdate/oai_sets.sql create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_set_mappings.tt create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_sets.tt create mode 100755 misc/migration_tools/build_oai_sets.pl diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 81da91b..7b090cf 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -35,6 +35,7 @@ use C4::Log; # logaction use C4::ClassSource; use C4::Charset; use C4::Linker; +use C4::OAI::Sets; use vars qw($VERSION @ISA @EXPORT); @@ -268,6 +269,11 @@ sub AddBiblio { # now add the record ModBiblioMarc( $record, $biblionumber, $frameworkcode ) unless $defer_marc_save; + # update OAI-PMH sets + if(C4::Context->preference("OAI-PMH:AutoUpdateSets")) { + C4::OAI::Sets::UpdateOAISetsBiblio($biblionumber, $record); + } + logaction( "CATALOGUING", "ADD", $biblionumber, "biblio" ) if C4::Context->preference("CataloguingLog"); return ( $biblionumber, $biblioitemnumber ); } @@ -339,6 +345,12 @@ sub ModBiblio { # modify the other koha tables _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode ); _koha_modify_biblioitem_nonmarc( $dbh, $oldbiblio ); + + # update OAI-PMH sets + if(C4::Context->preference("OAI-PMH:AutoUpdateSets")) { + C4::OAI::Sets::UpdateOAISetsBiblio($biblionumber, $record); + } + return 1; } diff --git a/C4/OAI/Sets.pm b/C4/OAI/Sets.pm new file mode 100644 index 0000000..8c28f7b --- /dev/null +++ b/C4/OAI/Sets.pm @@ -0,0 +1,589 @@ +package C4::OAI::Sets; + +# Copyright 2011 BibLibre +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +C4::OAI::Sets - OAI Sets management functions + +=head1 DESCRIPTION + +C4::OAI::Sets contains functions for managing storage and editing of OAI Sets. + +OAI Set description can be found L<here|http://www.openarchives.org/OAI/openarchivesprotocol.html#Set> + +=cut + +use Modern::Perl; +use C4::Context; + +use vars qw(@ISA @EXPORT); + +BEGIN { + require Exporter; + @ISA = qw(Exporter); + @EXPORT = qw( + &GetOAISets &GetOAISet &GetOAISetBySpec &ModOAISet &DelOAISet &AddOAISet + &GetOAISetsMappings &GetOAISetMappings &ModOAISetMappings + &GetOAISetsBiblio &ModOAISetsBiblios &AddOAISetsBiblios + &CalcOAISetsBiblio &UpdateOAISetsBiblio + ); +} + +=head1 FUNCTIONS + +=head2 GetOAISets + + $oai_sets = GetOAISets; + +GetOAISets return a array reference of hash references describing the sets. +The hash references looks like this: + + { + 'name' => 'set name', + 'spec' => 'set spec', + 'descriptions' => [ + 'description 1', + 'description 2', + ... + ] + } + +=cut + +sub GetOAISets { + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * FROM oai_sets + }; + my $sth = $dbh->prepare($query); + $sth->execute; + my $results = $sth->fetchall_arrayref({}); + + $query = qq{ + SELECT description + FROM oai_sets_descriptions + WHERE set_id = ? + }; + $sth = $dbh->prepare($query); + foreach my $set (@$results) { + $sth->execute($set->{'id'}); + my $desc = $sth->fetchall_arrayref({}); + foreach (@$desc) { + push @{$set->{'descriptions'}}, $_->{'description'}; + } + } + + return $results; +} + +=head2 GetOAISet + + $set = GetOAISet($set_id); + +GetOAISet returns a hash reference describing the set with the given set_id. + +See GetOAISets to see what the hash looks like. + +=cut + +sub GetOAISet { + my ($set_id) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * + FROM oai_sets + WHERE id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); + my $set = $sth->fetchrow_hashref; + + $query = qq{ + SELECT description + FROM oai_sets_descriptions + WHERE set_id = ? + }; + $sth = $dbh->prepare($query); + $sth->execute($set->{'id'}); + my $desc = $sth->fetchall_arrayref({}); + foreach (@$desc) { + push @{$set->{'descriptions'}}, $_->{'description'}; + } + + return $set; +} + +=head2 GetOAISetBySpec + + my $set = GetOAISetBySpec($setSpec); + +Returns a hash describing the set whose spec is $setSpec + +=cut + +sub GetOAISetBySpec { + my $setSpec = shift; + + return unless defined $setSpec; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * + FROM oai_sets + WHERE spec = ? + LIMIT 1 + }; + my $sth = $dbh->prepare($query); + $sth->execute($setSpec); + + return $sth->fetchrow_hashref; +} + +=head2 ModOAISet + + my $set = { + 'id' => $set_id, # mandatory + 'spec' => $spec, # mandatory + 'name' => $name, # mandatory + 'descriptions => \@descriptions, # optional, [] to remove descriptions + }; + ModOAISet($set); + +ModOAISet modify a set in the database. + +=cut + +sub ModOAISet { + my ($set) = @_; + + return unless($set && $set->{'spec'} && $set->{'name'}); + + if(!defined $set->{'id'}) { + warn "Set ID not defined, can't modify the set"; + return; + } + + my $dbh = C4::Context->dbh; + my $query = qq{ + UPDATE oai_sets + SET spec = ?, + name = ? + WHERE id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set->{'spec'}, $set->{'name'}, $set->{'id'}); + + if($set->{'descriptions'}) { + $query = qq{ + DELETE FROM oai_sets_descriptions + WHERE set_id = ? + }; + $sth = $dbh->prepare($query); + $sth->execute($set->{'id'}); + + if(scalar @{$set->{'descriptions'}} > 0) { + $query = qq{ + INSERT INTO oai_sets_descriptions (set_id, description) + VALUES (?,?) + }; + $sth = $dbh->prepare($query); + foreach (@{ $set->{'descriptions'} }) { + $sth->execute($set->{'id'}, $_) if $_; + } + } + } +} + +=head2 DelOAISet + + DelOAISet($set_id); + +DelOAISet remove the set with the given set_id + +=cut + +sub DelOAISet { + my ($set_id) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + DELETE oai_sets, oai_sets_descriptions, oai_sets_mappings + FROM oai_sets + LEFT JOIN oai_sets_descriptions ON oai_sets_descriptions.set_id = oai_sets.id + LEFT JOIN oai_sets_mappings ON oai_sets_mappings.set_id = oai_sets.id + WHERE oai_sets.id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); +} + +=head2 AddOAISet + + my $set = { + 'id' => $set_id, # mandatory + 'spec' => $spec, # mandatory + 'name' => $name, # mandatory + 'descriptions => \@descriptions, # optional + }; + my $set_id = AddOAISet($set); + +AddOAISet adds a new set and returns its id, or undef if something went wrong. + +=cut + +sub AddOAISet { + my ($set) = @_; + + return unless($set && $set->{'spec'} && $set->{'name'}); + + my $set_id; + my $dbh = C4::Context->dbh; + my $query = qq{ + INSERT INTO oai_sets (spec, name) + VALUES (?,?) + }; + my $sth = $dbh->prepare($query); + if( $sth->execute($set->{'spec'}, $set->{'name'}) ) { + $set_id = $dbh->last_insert_id(undef, undef, 'oai_sets', undef); + if($set->{'descriptions'}) { + $query = qq{ + INSERT INTO oai_sets_descriptions (set_id, description) + VALUES (?,?) + }; + $sth = $dbh->prepare($query); + foreach( @{ $set->{'descriptions'} } ) { + $sth->execute($set_id, $_) if $_; + } + } + } else { + warn "AddOAISet failed"; + } + + return $set_id; +} + +=head2 GetOAISetsMappings + + my $mappings = GetOAISetsMappings; + +GetOAISetsMappings returns mappings for all OAI Sets. + +Mappings define how biblios are categorized in sets. +A mapping is defined by three properties: + + { + marcfield => 'XXX', # the MARC field to check + marcsubfield => 'Y', # the MARC subfield to check + marcvalue => 'zzzz', # the value to check + } + +If defined in a set mapping, a biblio which have at least one 'Y' subfield of +one 'XXX' field equal to 'zzzz' will belong to this set. +If multiple mappings are defined in a set, the biblio will belong to this set +if at least one condition is matched. + +GetOAISetsMappings returns a hashref of arrayrefs of hashrefs. +The first hashref keys are the sets IDs, so it looks like this: + + $mappings = { + '1' => [ + { + marcfield => 'XXX', + marcsubfield => 'Y', + marcvalue => 'zzzz' + }, + { + ... + }, + ... + ], + '2' => [...], + ... + }; + +=cut + +sub GetOAISetsMappings { + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * FROM oai_sets_mappings + }; + my $sth = $dbh->prepare($query); + $sth->execute; + + my $mappings = {}; + while(my $result = $sth->fetchrow_hashref) { + push @{ $mappings->{$result->{'set_id'}} }, { + marcfield => $result->{'marcfield'}, + marcsubfield => $result->{'marcsubfield'}, + marcvalue => $result->{'marcvalue'} + }; + } + + return $mappings; +} + +=head2 GetOAISetMappings + + my $set_mappings = GetOAISetMappings($set_id); + +Return mappings for the set with given set_id. It's an arrayref of hashrefs + +=cut + +sub GetOAISetMappings { + my ($set_id) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * + FROM oai_sets_mappings + WHERE set_id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); + + my @mappings; + while(my $result = $sth->fetchrow_hashref) { + push @mappings, { + marcfield => $result->{'marcfield'}, + marcsubfield => $result->{'marcsubfield'}, + marcvalue => $result->{'marcvalue'} + }; + } + + return \@mappings; +} + +=head2 ModOAISetMappings { + + my $mappings = [ + { + marcfield => 'XXX', + marcsubfield => 'Y', + marcvalue => 'zzzz' + }, + ... + ]; + ModOAISetMappings($set_id, $mappings); + +ModOAISetMappings modifies mappings of a given set. + +=cut + +sub ModOAISetMappings { + my ($set_id, $mappings) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + DELETE FROM oai_sets_mappings + WHERE set_id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); + + if(scalar @$mappings > 0) { + $query = qq{ + INSERT INTO oai_sets_mappings (set_id, marcfield, marcsubfield, marcvalue) + VALUES (?,?,?,?) + }; + $sth = $dbh->prepare($query); + foreach (@$mappings) { + $sth->execute($set_id, $_->{'marcfield'}, $_->{'marcsubfield'}, $_->{'marcvalue'}); + } + } +} + +=head2 GetOAISetsBiblio + + $oai_sets = GetOAISetsBiblio($biblionumber); + +Return the OAI sets where biblio appears. + +Return value is an arrayref of hashref where each element of the array is a set. +Keys of hash are id, spec and name + +=cut + +sub GetOAISetsBiblio { + my ($biblionumber) = @_; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT oai_sets.* + FROM oai_sets + LEFT JOIN oai_sets_biblios ON oai_sets_biblios.set_id = oai_sets.id + WHERE biblionumber = ? + }; + my $sth = $dbh->prepare($query); + + $sth->execute($biblionumber); + return $sth->fetchall_arrayref({}); +} + +=head2 DelOAISetsBiblio + + DelOAISetsBiblio($biblionumber); + +Remove a biblio from all sets + +=cut + +sub DelOAISetsBiblio { + my ($biblionumber) = @_; + + return unless $biblionumber; + + my $dbh = C4::Context->dbh; + my $query = qq{ + DELETE FROM oai_sets_biblios + WHERE biblionumber = ? + }; + my $sth = $dbh->prepare($query); + return $sth->execute($biblionumber); +} + +=head2 CalcOAISetsBiblio + + my @sets = CalcOAISetsBiblio($record, $oai_sets_mappings); + +Return a list of set ids the record belongs to. $record must be a MARC::Record +and $oai_sets_mappings (optional) must be a hashref returned by +GetOAISetsMappings + +=cut + +sub CalcOAISetsBiblio { + my ($record, $oai_sets_mappings) = @_; + + return unless $record; + + $oai_sets_mappings ||= GetOAISetsMappings; + + my @biblio_sets; + foreach my $set_id (keys %$oai_sets_mappings) { + foreach my $mapping (@{ $oai_sets_mappings->{$set_id} }) { + next if not $mapping; + my $field = $mapping->{'marcfield'}; + my $subfield = $mapping->{'marcsubfield'}; + my $value = $mapping->{'marcvalue'}; + + my @subfield_values = $record->subfield($field, $subfield); + if(0 < grep /^$value$/, @subfield_values) { + push @biblio_sets, $set_id; + last; + } + } + } + return @biblio_sets; +} + +=head2 ModOAISetsBiblios + + my $oai_sets_biblios = { + '1' => [1, 3, 4], # key is the set_id, and value is an array ref of biblionumbers + '2' => [], + ... + }; + ModOAISetsBiblios($oai_sets_biblios); + +ModOAISetsBiblios truncate oai_sets_biblios table and call AddOAISetsBiblios. +This table is then used in opac/oai.pl. + +=cut + +sub ModOAISetsBiblios { + my $oai_sets_biblios = shift; + + return unless ref($oai_sets_biblios) eq "HASH"; + + my $dbh = C4::Context->dbh; + my $query = qq{ + TRUNCATE TABLE oai_sets_biblios + }; + my $sth = $dbh->prepare($query); + $sth->execute; + AddOAISetsBiblios($oai_sets_biblios); +} + +=head2 UpdateOAISetsBiblio + + UpdateOAISetsBiblio($biblionumber, $record); + +Update OAI sets for one biblio. The two parameters are mandatory. +$record is a MARC::Record. + +=cut + +sub UpdateOAISetsBiblio { + my ($biblionumber, $record) = @_; + + return unless($biblionumber and $record); + + my $sets_biblios; + my @sets = CalcOAISetsBiblio($record); + foreach (@sets) { + push @{ $sets_biblios->{$_} }, $biblionumber; + } + DelOAISetsBiblio($biblionumber); + AddOAISetsBiblios($sets_biblios); +} + +=head2 AddOAISetsBiblios + + my $oai_sets_biblios = { + '1' => [1, 3, 4], # key is the set_id, and value is an array ref of biblionumbers + '2' => [], + ... + }; + ModOAISetsBiblios($oai_sets_biblios); + +AddOAISetsBiblios insert given infos in oai_sets_biblios table. +This table is then used in opac/oai.pl. + +=cut + +sub AddOAISetsBiblios { + my $oai_sets_biblios = shift; + + return unless ref($oai_sets_biblios) eq "HASH"; + + my $dbh = C4::Context->dbh; + my $query = qq{ + INSERT INTO oai_sets_biblios (set_id, biblionumber) + VALUES (?,?) + }; + my $sth = $dbh->prepare($query); + foreach my $set_id (keys %$oai_sets_biblios) { + foreach my $biblionumber (@{$oai_sets_biblios->{$set_id}}) { + $sth->execute($set_id, $biblionumber); + } + } +} + +1; diff --git a/admin/oai_set_mappings.pl b/admin/oai_set_mappings.pl new file mode 100755 index 0000000..4d570f9 --- /dev/null +++ b/admin/oai_set_mappings.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +# Copyright 2011 BibLibre SARL +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +oai_set_mappings.pl + +=head1 DESCRIPTION + +Define mappings for a given set. +Mappings are conditions that define which biblio is included in which set. +A condition is in the form 200$a = 'abc'. +Multiple conditions can be defined for a given set. In this case, +the OR operator will be applied. + +=cut + +use Modern::Perl; + +use CGI; +use C4::Auth; +use C4::Output; +use C4::OAI::Sets; + +use Data::Dumper; + +my $input = new CGI; +my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( { + template_name => 'admin/oai_set_mappings.tt', + query => $input, + type => 'intranet', + authnotrequired => 0, + flagsrequired => { 'parameters' => '*' }, + debug => 1, +} ); + +my $id = $input->param('id'); +my $op = $input->param('op'); + +if($op && $op eq "save") { + my @marcfields = $input->param('marcfield'); + my @marcsubfields = $input->param('marcsubfield'); + my @marcvalues = $input->param('marcvalue'); + + my @mappings; + my $i = 0; + while($i < @marcfields and $i < @marcsubfields and $i < @marcvalues) { + if($marcfields[$i] and $marcsubfields[$i] and $marcvalues[$i]) { + push @mappings, { + marcfield => $marcfields[$i], + marcsubfield => $marcsubfields[$i], + marcvalue => $marcvalues[$i] + }; + } + $i++; + } + ModOAISetMappings($id, \@mappings); + $template->param(mappings_saved => 1); +} + +my $set = GetOAISet($id); +my $mappings = GetOAISetMappings($id); + +$template->param( + id => $id, + setName => $set->{'name'}, + setSpec => $set->{'spec'}, + mappings => $mappings, +); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/admin/oai_sets.pl b/admin/oai_sets.pl new file mode 100755 index 0000000..a826107 --- /dev/null +++ b/admin/oai_sets.pl @@ -0,0 +1,102 @@ +#!/usr/bin/perl + +# Copyright 2011 BibLibre SARL +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +oai_sets.pl + +=head1 DESCRIPTION + +Admin page to describe OAI SETs + +=cut + +use Modern::Perl; + +use CGI; +use C4::Auth; +use C4::Output; +use C4::OAI::Sets; + +use Data::Dumper; + +my $input = new CGI; +my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( { + template_name => 'admin/oai_sets.tt', + query => $input, + type => 'intranet', + authnotrequired => 0, + flagsrequired => { 'parameters' => '*' }, + debug => 1, +} ); + +my $op = $input->param('op'); + +if($op && $op eq "new") { + $template->param( op_new => 1 ); +} elsif($op && $op eq "savenew") { + my $spec = $input->param('spec'); + my $name = $input->param('name'); + my @descriptions = $input->param('description'); + AddOAISet({ + spec => $spec, + name => $name, + descriptions => \@descriptions + }); +} elsif($op && $op eq "mod") { + my $id = $input->param('id'); + my $set = GetOAISet($id); + $template->param( + op_mod => 1, + id => $set->{'id'}, + spec => $set->{'spec'}, + name => $set->{'name'}, + descriptions => [ map { {description => $_} } @{ $set->{'descriptions'} } ], + ); +} elsif($op && $op eq "savemod") { + my $id = $input->param('id'); + my $spec = $input->param('spec'); + my $name = $input->param('name'); + my @descriptions = $input->param('description'); + ModOAISet({ + id => $id, + spec => $spec, + name => $name, + descriptions => \@descriptions + }); +} elsif($op && $op eq "del") { + my $id = $input->param('id'); + DelOAISet($id); +} + +my $OAISets = GetOAISets; +my @sets_loop; +foreach(@$OAISets) { + push @sets_loop, { + id => $_->{'id'}, + spec => $_->{'spec'}, + name => $_->{'name'}, + descriptions => [ map { {description => $_} } @{ $_->{'descriptions'} } ] + }; +} + +$template->param( + sets_loop => \@sets_loop, +); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/installer/data/mysql/atomicupdate/oai_sets.sql b/installer/data/mysql/atomicupdate/oai_sets.sql new file mode 100644 index 0000000..a843ab7 --- /dev/null +++ b/installer/data/mysql/atomicupdate/oai_sets.sql @@ -0,0 +1,35 @@ +DROP TABLE IF EXISTS `oai_sets_descriptions`; +DROP TABLE IF EXISTS `oai_sets_mappings`; +DROP TABLE IF EXISTS `oai_sets_biblios`; +DROP TABLE IF EXISTS `oai_sets`; + +CREATE TABLE `oai_sets` ( + `id` int(11) NOT NULL auto_increment, + `spec` varchar(80) NOT NULL UNIQUE, + `name` varchar(80) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `oai_sets_descriptions` ( + `set_id` int(11) NOT NULL, + `description` varchar(255) NOT NULL, + CONSTRAINT `oai_sets_descriptions_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `oai_sets_mappings` ( + `set_id` int(11) NOT NULL, + `marcfield` char(3) NOT NULL, + `marcsubfield` char(1) NOT NULL, + `marcvalue` varchar(80) NOT NULL, + CONSTRAINT `oai_sets_mappings_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `oai_sets_biblios` ( + `biblionumber` int(11) NOT NULL, + `set_id` int(11) NOT NULL, + PRIMARY KEY (`biblionumber`, `set_id`), + CONSTRAINT `oai_sets_biblios_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `oai_sets_biblios_ibfk_2` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('OAI-PMH:AutoUpdateSets','0','Automatically update OAI sets when a bibliographic record is created or updated','','YesNo'); diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 597a8be..ab23768 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -1354,6 +1354,55 @@ CREATE TABLE `nozebra` ( ENGINE=InnoDB DEFAULT CHARSET=utf8; -- +-- Table structure for table `oai_sets` +-- + +DROP TABLE IF EXISTS `oai_sets`; +CREATE TABLE `oai_sets` ( + `id` int(11) NOT NULL auto_increment, + `spec` varchar(80) NOT NULL UNIQUE, + `name` varchar(80) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_sets_descriptions` +-- + +DROP TABLE IF EXISTS `oai_sets_descriptions`; +CREATE TABLE `oai_sets_descriptions` ( + `set_id` int(11) NOT NULL, + `description` varchar(255) NOT NULL, + CONSTRAINT `oai_sets_descriptions_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_sets_mappings` +-- + +DROP TABLE IF EXISTS `oai_sets_mappings`; +CREATE TABLE `oai_sets_mappings` ( + `set_id` int(11) NOT NULL, + `marcfield` char(3) NOT NULL, + `marcsubfield` char(1) NOT NULL, + `marcvalue` varchar(80) NOT NULL, + CONSTRAINT `oai_sets_mappings_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_sets_biblios` +-- + +DROP TABLE IF EXISTS `oai_sets_biblios`; +CREATE TABLE `oai_sets_biblios` ( + `biblionumber` int(11) NOT NULL, + `set_id` int(11) NOT NULL, + PRIMARY KEY (`biblionumber`, `set_id`), + CONSTRAINT `oai_sets_biblios_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `oai_sets_biblios_ibfk_2` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- -- Table structure for table `old_issues` -- diff --git a/installer/data/mysql/sysprefs.sql b/installer/data/mysql/sysprefs.sql index bda97f5..aa0a101 100644 --- a/installer/data/mysql/sysprefs.sql +++ b/installer/data/mysql/sysprefs.sql @@ -344,3 +344,4 @@ INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES(' INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('LinkerRelink',1,'If ON the authority linker will relink headings that have previously been linked every time it runs.',NULL,'YesNo'); INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('LinkerKeepStale',0,'If ON the authority linker will keep existing authority links for headings where it is unable to find a match.',NULL,'YesNo'); INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('CatalogModuleRelink',0,'If OFF the linker will never replace the authids that are set in the cataloging module.',NULL,'YesNo'); +INSERT INTO systempreferences (variable,value,explanation,options,type) VALUES('OAI-PMH:AutoUpdateSets','0','Automatically update OAI sets when a bibliographic record is created or updated','','YesNo'); diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index 33a3df6..954612f 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -4765,6 +4765,16 @@ if (C4::Context->preference("Version") < TransformToNum($DBversion)) { SetVersion($DBversion); } +$DBversion = "XXX"; +if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { + my $installer = C4::Installer->new(); + my $full_path = C4::Context->config('intranetdir') . "/installer/data/$installer->{dbms}/atomicupdate/oai_sets.sql"; + my $error = $installer->load_sql($full_path); + warn $error if $error; + print "Upgrade to $DBversion done (Atomic update for OAI-PMH sets management)\n"; + SetVersion($DBversion); +} + =head1 FUNCTIONS =head2 DropAllForeignKeys($table) diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index e9537f4..74dd667 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -43,6 +43,7 @@ <li><a href="/cgi-bin/koha/admin/authtypes.pl">Authority types</a></li> <li><a href="/cgi-bin/koha/admin/classsources.pl">Classification sources</a></li> <li><a href="/cgi-bin/koha/admin/matching-rules.pl">Record matching rules</a></li> + <li><a href="/cgi-bin/koha/admin/oai_sets.pl">OAI Sets configuration</a></li> </ul> <h5>Acquisition parameters</h5> diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index fe76ae8..cea2215 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -74,6 +74,8 @@ <dd>Define classification sources (i.e., call number schemes) used by your collection. Also define filing rules used for sorting call numbers.</dd> <dt><a href="/cgi-bin/koha/admin/matching-rules.pl">Record matching rules</a></dt> <dd>Manage rules for automatically matching MARC records during record imports.</dd> + <dt><a href="/cgi-bin/koha/admin/oai_sets.pl">OAI Sets Configuration</a></dt> + <dd>Manage OAI Sets</dd> </dl> <h3>Acquisition parameters</h3> diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt new file mode 100644 index 0000000..894d7d5 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt @@ -0,0 +1,103 @@ +[% INCLUDE 'doc-head-open.inc' %] +<title>Koha › Admin › OAI Set Mappings</title> +[% INCLUDE 'doc-head-close.inc' %] +<script type="text/javascript"> +//<![CDATA[ +$(document).ready(function() { + // Some JS +}); + +function newCondition() { + var tr = $('#ORbutton').parents('tr'); + var clone = $(tr).clone(); + $("#ORbutton").parent('td').replaceWith('<td style="text-align:center">OR</td>'); + $(tr).parent('tbody').append(clone); +} + +function hideDialogBox() { + $('div.dialog').remove(); +} + +function returnToSetsPage() { + window.location.href = "/cgi-bin/koha/admin/oai_sets.pl"; +} +//]]> +</script> +</head> + +<body> +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + +<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> › <a href="/cgi-bin/koha/admin/admin-home.pl">Admin</a> › <a href="/cgi-bin/koha/admin/oai_set_mappings.pl?id=[% id %]">OAI Set Mappings</a></div> + +<div id="doc3" class="yui-t2"> + +<div id="bd"> + <div id="yui-main"> + <div class="yui-b"> + [% IF ( mappings_saved ) %] + <div class="dialog"> + <p>Mappings have been saved</p> + <p><a href="/cgi-bin/koha/admin/oai_sets.pl">Return to sets management</a></p> + </div> + [% END %] + <h1>Mappings for set '[% setName %]' ([% setSpec %])</h1> + [% UNLESS ( mappings ) %] + <p class="warning">Warning: no mappings defined for this set</p> + [% END %] + <form action="/cgi-bin/koha/admin/oai_set_mappings.pl" method="post" onsubmit="hideDialogBox();"> + <table id="mappings"> + <thead> + <tr> + <th>Field</th> + <th>Subfield</th> + <th> </th> + <th>Value</th> + <th> </th> + </tr> + </thead> + <tbody> + [% IF ( mappings ) %] + [% FOREACH mapping IN mappings %] + <tr> + <td><input type="text" name="marcfield" size="3" value="[% mapping.marcfield %]" /></td> + <td style="text-align:center"><input type="text" name="marcsubfield" size="1" value="[% mapping.marcsubfield %]" /></td> + <td>is equal to</td> + <td><input type="text" name="marcvalue" value="[% mapping.marcvalue %]" /></td> + <td style="text-align:center"> + [% IF ( loop.last ) %] + <input type="button" id="ORbutton" value="OR" onclick="newCondition()"/> + [% ELSE %] + OR + [% END %] + </td> + </tr> + [% END %] + [% ELSE %] + <tr> + <td><input type="text" name="marcfield" size="3" /></td> + <td style="text-align:center"><input type="text" name="marcsubfield" size="1" /></td> + <td>is equal to</td> + <td><input type="text" name="marcvalue" /></td> + <td><input type="button" id="ORbutton" value="OR" onclick="newCondition()"/></td> + </tr> + [% END %] + </tbody> + </table> + <p class="hint">Hint: to delete a line, empty at least one of the text fields in this line</p> + <input type="hidden" name="id" value="[% id %]" /> + <input type="hidden" name="op" value="save" /> + <fieldset class="action"> + <input type="submit" value="Save" /> + <input type="button" value="Cancel" onclick="returnToSetsPage();" /> + </fieldset> + </form> + + </div> + </div> + <div class="yui-b"> + [% INCLUDE 'admin-menu.inc' %] + </div> +</div> +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt new file mode 100644 index 0000000..444f972 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt @@ -0,0 +1,140 @@ +[% INCLUDE 'doc-head-open.inc' %] +<title>Koha › Admin › OAI Sets</title> +[% INCLUDE 'doc-head-close.inc' %] +<script type="text/javascript"> +//<![CDATA[ +function newDescField() { + $("#descriptionlist").append( + '<li>' + + '<textarea style="vertical-align:middle" name="description"></textarea>' + + '<a style="cursor:pointer" onclick="delDescField(this)"> ×</a>' + + '</li>' + ); +} + +function delDescField(minusButton) { + var li = $(minusButton).parent('li'); + $(li).remove(); +} + +$(document).ready(function() { + // Some JS +}); +//]]> +</script> +</head> + +<body> +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + +<div id="breadcrumbs"><a href="/cgi-bin/koha/mainpage.pl">Home</a> › <a href="/cgi-bin/koha/admin/admin-home.pl">Admin</a> › OAI Sets</div> + +<div id="doc3" class="yui-t2"> + +<div id="bd"> + <div id="yui-main"> + <div class="yui-b"> + <h1>OAI Sets Configuration</h1> + + [% IF op_new %] + <h2>Add a new set</h2> + <form method="post" action="/cgi-bin/koha/admin/oai_sets.pl"> + <input type="hidden" name="op" value="savenew" /> + <fieldset> + <label for="spec">setSpec</label> + <input type="text" id="spec" name="spec" /> + <br /> + <label for="name">setName</label> + <input type="text" id="name" name="name" /> + <br /> + <label>setDescriptions</label> + <ul id="descriptionlist"> + </ul> + <a style="cursor:pointer" onclick='newDescField()'>Add description</a> + </fieldset> + <input type="submit" value="Save" /> + <input type="button" value="Cancel" onclick="window.location.href = '/cgi-bin/koha/admin/oai_sets.pl'" /> + </form> + [% ELSE %][% IF op_mod %] + <h2>Modify set '[% spec %]'</h2> + <form method="post" action="/cgi-bin/koha/admin/oai_sets.pl"> + <input type="hidden" name="op" value="savemod" /> + <input type="hidden" name="id" value="[% id %]" /> + <fieldset> + <label for="spec">setSpec</label> + <input type="text" id="spec" name="spec" value="[% spec %]" /> + <br /> + <label for="name">setName</label> + <input type="text" id="name" name="name" value="[% name %]" /> + <br /> + <label>setDescriptions</label> + <ul id="descriptionlist"> + [% FOREACH desc IN descriptions %] + <li> + <textarea style="vertical-align:middle" name="description">[% desc.description %]</textarea> + <a style="cursor:pointer" onclick="delDescField(this)"> ×</a> + </li> + [% END %] + </ul> + <a style="cursor:pointer" onclick='newDescField()'>Add description</a> + </fieldset> + <input type="submit" value="Save" /> + <input type="button" value="Cancel" onclick="window.location.href = '/cgi-bin/koha/admin/oai_sets.pl'" /> + </form> + [% END %] + [% END %] + + <h2>List of sets</h2> + [% UNLESS ( op_new ) %] + <a href="/cgi-bin/koha/admin/oai_sets.pl?op=new">Add a new set</a> + [% END %] + [% IF sets_loop %] + <table> + <thead> + <tr> + <th>setSpec</th> + <th>setName</th> + <th>setDescriptions</th> + <th>Action</th> + </tr> + </thead> + <tbody> + [% FOREACH set IN sets_loop %] + <tr> + <td>[% set.spec %]</td> + <td>[% set.name %]</td> + <td> + [% IF set.descriptions %] + <ul> + [% FOREACH desc IN set.descriptions %] + <li>[% desc.description %]</li> + [% END %] + </ul> + [% ELSE %] + <em>No descriptions</em> + [% END %] + </td> + <td> + <a href="/cgi-bin/koha/admin/oai_sets.pl?op=mod&id=[% set.id %]">Modify</a> + | + <a href="/cgi-bin/koha/admin/oai_sets.pl?op=del&id=[% set.id %]">Delete</a> + | + <a href="/cgi-bin/koha/admin/oai_set_mappings.pl?id=[% set.id %]">Define mappings</a> + </td> + </tr> + [% END %] + </tbody> + </table> + [% ELSE %] + <p>There is no set defined.</p> + [% END %] + + + </div> + </div> + <div class="yui-b"> + [% INCLUDE 'admin-menu.inc' %] + </div> +</div> +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref index f4f5d6f..fae1fa9 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/web_services.pref @@ -21,6 +21,12 @@ Web Services: - pref: "OAI-PMH:ConfFile" class: file - . If empty, Koha OAI Server operates in normal mode, otherwise it operates in extended mode. In extended mode, it's possible to parameter other formats than marcxml or Dublin Core. OAI-PMH:ConfFile specify a YAML configuration file which list available metadata formats and XSL file used to create them from marcxml records. + - + - pref: "OAI-PMH:AutoUpdateSets" + choices: + yes: Enable + no: Disable + - automatic update of OAI-PMH sets when a bibliographic record is created or updated ILS-DI: - - pref: ILS-DI diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_set_mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_set_mappings.tt new file mode 100644 index 0000000..39fd21b --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_set_mappings.tt @@ -0,0 +1,40 @@ +[% INCLUDE 'help-top.inc' %] + +<h1>OAI-PMH Sets Mappings Configuration</h1> + +<p> + Here you can define how a set will be build (what records will belong to + this set) by defining mappings. Mappings are a list of conditions on record + content. A record only need to match one condition to belong to the set. +<p> + +<h2>Defining a mapping</h2> +<ol> + <li> + Fill the fields 'Field', 'Subfield' and 'Value'. For example if you + want to include in this set all records that have a 999$9 equal to + 'XXX'. Fill 'Field' with 999, 'Subfield' with 9 and 'Value' with XXX. + </li> + <li> + If you want to add another condition, click on 'OR' button and repeat + step 1. + </li> + <li>Click on 'Save'</li> +</ol> + +<p> + To delete a condition, just leave at least one of 'Field', 'Subfield' or + 'Value' empty and click on 'Save'. +</p> + +<p> + Note: Actually, a condition is true if value in the corresponding subfield + is strictly equal to what is defined if 'Value'. A record having + 999$9 = 'XXX YYY' will not belong to a set where condition is + 999$9 = 'XXX'. + <br /> + And it is case sensitive : a record having 999$9 = 'xxx' will not belong + to a set where condition is 999$9 = 'XXX'. +</p> + +[% INCLUDE 'help-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_sets.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_sets.tt new file mode 100644 index 0000000..7fc46cc --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/help/admin/oai_sets.tt @@ -0,0 +1,49 @@ +[% INCLUDE 'help-top.inc' %] + +<h1>OAI-PMH Sets Configuration</h1> + +<p>On this page you can create, modify and delete OAI-PMH sets<p> + +<h2>Create a set</h2> + +<ol> + <li>Click on the link 'Add a new set'</li> + <li>Fill the mandatory fields 'setSpec' and 'setName'</li> + <li> + Then you can add descriptions for this set. To do this click on + 'Add description' and fill the newly created text box. You can add as + many descriptions as you want. + </li> + <li>Click on 'Save' button'</li> +</ol> + +<h2>Modify a set</h2> + +<p> + To modify a set, just click on the link 'Modify' on the same line of the + set you want to modify. A form similar to set creation form will appear and + allow you to modify the setSpec, setName and descriptions. +<p> + +<h2>Delete a set</h2> + +<p> + To delete a set, just click on the link 'Delete' on the same line of the + set you want to delete. +</p> + +<h2>Define mappings</h2> + +<p> + The 'Define mappings' link allow you to tell how the set will be build + (what records will belong to this set) +</p> + +<h2>Build sets</h2> + +<p> + Once you have configured all your sets, you have to build the sets. This is + done by calling the script misc/migration_tools/build_oai_sets.pl. +</p> + +[% INCLUDE 'help-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl index 6352384..a75b800 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl @@ -163,9 +163,10 @@ </dc:identifier> </xsl:for-each> <xsl:for-each select="marc:datafield[@tag=090]"> - <dc:identifier> - <xsl:text>http://opac.mylibrary.org/bib/</xsl:text> - <xsl:value-of select="marc:subfield[@code='a']"/> + <dc:identifier> + <xsl:value-of select="$OPACBaseURL" /> + <xsl:text>/bib/</xsl:text> + <xsl:value-of select="marc:subfield[@code='a']"/> </dc:identifier> </xsl:for-each> <xsl:for-each select="marc:datafield[@tag=995]"> @@ -175,10 +176,10 @@ <xsl:when test="marc:subfield[@code='c']='MAIN'">Main Branch</xsl:when> <xsl:when test="marc:subfield[@code='c']='BIB2'">Library 2</xsl:when> </xsl:choose> - <xsl:foreach select="marc:subfield[@code='k']"> + <xsl:for-each select="marc:subfield[@code='k']"> <xsl:text>:</xsl:text> <xsl:value-of select="."/> - </xsl:foreach> + </xsl:for-each> </dc:identifier> </xsl:for-each> </xsl:template> diff --git a/misc/migration_tools/build_oai_sets.pl b/misc/migration_tools/build_oai_sets.pl new file mode 100755 index 0000000..577c06e --- /dev/null +++ b/misc/migration_tools/build_oai_sets.pl @@ -0,0 +1,165 @@ +#!/usr/bin/perl + +# Copyright 2011 BibLibre +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 DESCRIPTION + +This script build OAI-PMH sets (to be used by opac/oai.pl) according to sets +and mappings defined in Koha. It reads informations from oai_sets and +oai_sets_mappings, and then fill table oai_sets_biblios with builded infos. + +=head1 USAGE + + build_oai_sets.pl [-h] [-v] [-r] [-i] [-l LENGTH [-o OFFSET]] + -h Print help message; + -v Be verbose + -r Truncate table oai_sets_biblios before inserting new rows + -i Embed items informations, mandatory if you defined mappings + on item fields + -l LENGTH Process LENGTH biblios + -o OFFSET If LENGTH is defined, start processing from OFFSET + +=cut + +use Modern::Perl; +use MARC::Record; +use MARC::File::XML; +use List::MoreUtils qw/uniq/; +use Getopt::Std; + +use C4::Context; +use C4::Charset qw/StripNonXmlChars/; +use C4::Biblio; +use C4::OAI::Sets; + +my %opts; +$Getopt::Std::STANDARD_HELP_VERSION = 1; +my $go = getopts('vo:l:ihr', \%opts); + +if(!$go or $opts{h}){ + &print_usage; + exit; +} + +my $verbose = $opts{v}; +my $offset = $opts{o}; +my $length = $opts{l}; +my $embed_items = $opts{i}; +my $reset = $opts{r}; + +my $dbh = C4::Context->dbh; + +# Get OAI sets mappings +my $mappings = GetOAISetsMappings; + +# Get all biblionumbers and marcxml +print "Retrieving biblios... " if $verbose; +my $query = qq{ + SELECT biblionumber, marcxml + FROM biblioitems +}; +if($length) { + $query .= "LIMIT $length"; + if($offset) { + $query .= " OFFSET $offset"; + } +} +my $sth = $dbh->prepare($query); +$sth->execute; +my $results = $sth->fetchall_arrayref({}); +print "done.\n" if $verbose; + +# Build lists of parents sets +my $sets = GetOAISets; +my $parentsets; +foreach my $set (@$sets) { + my $setSpec = $set->{'spec'}; + while($setSpec =~ /^(.+):(.+)$/) { + my $parent = $1; + my $parent_set = GetOAISetBySpec($parent); + if($parent_set) { + push @{ $parentsets->{$set->{'id'}} }, $parent_set->{'id'}; + $setSpec = $parent; + } else { + last; + } + } +} + +my $num_biblios = scalar @$results; +my $i = 1; +my $sets_biblios = {}; +foreach my $res (@$results) { + my $biblionumber = $res->{'biblionumber'}; + my $marcxml = $res->{'marcxml'}; + if($verbose and $i % 1000 == 0) { + my $percent = ($i * 100) / $num_biblios; + $percent = sprintf("%.2f", $percent); + say "Progression: $i/$num_biblios ($percent %)"; + } + # The following lines are copied from GetMarcBiblio + # We don't call GetMarcBiblio to avoid a sql query to be executed each time + $marcxml = StripNonXmlChars($marcxml); + MARC::File::XML->default_record_format(C4::Context->preference('marcflavour')); + my $record; + eval { + $record = MARC::Record::new_from_xml($marcxml, "utf8", C4::Context->preference('marcflavour')); + }; + if($@) { + warn "(biblio $biblionumber) Error while creating record from marcxml: $@"; + next; + } + if($embed_items) { + C4::Biblio::EmbedItemsInMarcBiblio($record, $biblionumber); + } + + my @biblio_sets = CalcOAISetsBiblio($record, $mappings); + foreach my $set_id (@biblio_sets) { + push @{ $sets_biblios->{$set_id} }, $biblionumber; + foreach my $parent_set_id ( @{ $parentsets->{$set_id} } ) { + push @{ $sets_biblios->{$parent_set_id} }, $biblionumber; + } + } + $i++; +} +say "Progression: done." if $verbose; + +say "Summary:"; +foreach my $set_id (keys %$sets_biblios) { + $sets_biblios->{$set_id} = [ uniq @{ $sets_biblios->{$set_id} } ]; + my $set = GetOAISet($set_id); + my $setSpec = $set->{'spec'}; + say "Set '$setSpec': ". scalar(@{$sets_biblios->{$set_id}}) ." biblios"; +} + +print "Updating database... "; +if($reset) { + ModOAISetsBiblios( {} ); +} +AddOAISetsBiblios($sets_biblios); +print "done.\n"; + +sub print_usage { + print "build_oai_sets.pl: Build OAI-PMH sets, according to mappings defined in Koha\n"; + print "Usage: build_oai_sets.pl [-h] [-v] [-i] [-l LENGTH [-o OFFSET]]\n\n"; + print "\t-h\t\tPrint this help and exit\n"; + print "\t-v\t\tBe verbose\n"; + print "\t-i\t\tEmbed items informations, mandatory if you defined mappings on item fields\n"; + print "\t-l LENGTH\tProcess LENGTH biblios\n"; + print "\t-o OFFSET\tIf LENGTH is defined, start processing from OFFSET\n\n"; +} diff --git a/opac/oai.pl b/opac/oai.pl index 2ef3f28..bad250a 100755 --- a/opac/oai.pl +++ b/opac/oai.pl @@ -2,7 +2,6 @@ use strict; use warnings; -use diagnostics; use CGI qw/:standard -oldstyle_urls/; use vars qw( $GZIP ); @@ -59,7 +58,6 @@ package C4::OAI::ResumptionToken; use strict; use warnings; -use diagnostics; use HTTP::OAI; use base ("HTTP::OAI::ResumptionToken"); @@ -70,9 +68,9 @@ sub new { my $self = $class->SUPER::new(%args); - my ($metadata_prefix, $offset, $from, $until); + my ($metadata_prefix, $offset, $from, $until, $set); if ( $args{ resumptionToken } ) { - ($metadata_prefix, $offset, $from, $until) + ($metadata_prefix, $offset, $from, $until, $set) = split( ':', $args{resumptionToken} ); } else { @@ -84,15 +82,17 @@ sub new { $until = sprintf( "%.4d-%.2d-%.2d", $year+1900, $mon+1,$mday ); } $offset = $args{ offset } || 0; + $set = $args{set}; } $self->{ metadata_prefix } = $metadata_prefix; $self->{ offset } = $offset; $self->{ from } = $from; $self->{ until } = $until; + $self->{ set } = $set; $self->resumptionToken( - join( ':', $metadata_prefix, $offset, $from, $until ) ); + join( ':', $metadata_prefix, $offset, $from, $until, $set ) ); $self->cursor( $offset ); return $self; @@ -106,7 +106,6 @@ package C4::OAI::Identify; use strict; use warnings; -use diagnostics; use HTTP::OAI; use C4::Context; @@ -145,7 +144,6 @@ package C4::OAI::ListMetadataFormats; use strict; use warnings; -use diagnostics; use HTTP::OAI; use base ("HTTP::OAI::ListMetadataFormats"); @@ -188,14 +186,13 @@ package C4::OAI::Record; use strict; use warnings; -use diagnostics; use HTTP::OAI; use HTTP::OAI::Metadata::OAI_DC; use base ("HTTP::OAI::Record"); sub new { - my ($class, $repository, $marcxml, $timestamp, %args) = @_; + my ($class, $repository, $marcxml, $timestamp, $setSpecs, %args) = @_; my $self = $class->SUPER::new(%args); @@ -205,11 +202,18 @@ sub new { datestamp => $timestamp, ) ); + foreach my $setSpec (@$setSpecs) { + $self->header->setSpec($setSpec); + } + my $parser = XML::LibXML->new(); my $record_dom = $parser->parse_string( $marcxml ); my $format = $args{metadataPrefix}; if ( $format ne 'marcxml' ) { - $record_dom = $repository->stylesheet($format)->transform( $record_dom ); + my %args = ( + OPACBaseURL => "'" . C4::Context->preference('OPACBaseURL') . "'" + ); + $record_dom = $repository->stylesheet($format)->transform($record_dom, %args); } $self->metadata( HTTP::OAI::Metadata->new( dom => $record_dom ) ); @@ -224,8 +228,8 @@ package C4::OAI::GetRecord; use strict; use warnings; -use diagnostics; use HTTP::OAI; +use C4::OAI::Sets; use base ("HTTP::OAI::GetRecord"); @@ -254,9 +258,15 @@ sub new { ); } + my $oai_sets = GetOAISetsBiblio($biblionumber); + my @setSpecs; + foreach (@$oai_sets) { + push @setSpecs, $_->{spec}; + } + #$self->header( HTTP::OAI::Header->new( identifier => $args{identifier} ) ); $self->record( C4::OAI::Record->new( - $repository, $marcxml, $timestamp, %args ) ); + $repository, $marcxml, $timestamp, \@setSpecs, %args ) ); return $self; } @@ -269,8 +279,8 @@ package C4::OAI::ListIdentifiers; use strict; use warnings; -use diagnostics; use HTTP::OAI; +use C4::OAI::Sets; use base ("HTTP::OAI::ListIdentifiers"); @@ -282,42 +292,148 @@ sub new { my $token = new C4::OAI::ResumptionToken( %args ); my $dbh = C4::Context->dbh; - my $sql = "SELECT biblionumber, timestamp - FROM biblioitems - WHERE timestamp >= ? AND timestamp <= ? - LIMIT " . $repository->{koha_max_count} . " - OFFSET " . $token->{offset}; + my $set; + if(defined $token->{'set'}) { + $set = GetOAISetBySpec($token->{'set'}); + } + my $sql = " + SELECT biblioitems.biblionumber, biblioitems.timestamp + FROM biblioitems + "; + $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; + $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; + $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; + $sql .= " + LIMIT $repository->{'koha_max_count'} + OFFSET $token->{'offset'} + "; my $sth = $dbh->prepare( $sql ); - $sth->execute( $token->{from}, $token->{until} ); + my @bind_params = ($token->{'from'}, $token->{'until'}); + push @bind_params, $set->{'id'} if defined $set; + $sth->execute( @bind_params ); my $pos = $token->{offset}; - while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { - $timestamp =~ s/ /T/, $timestamp .= 'Z'; + while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { + $timestamp =~ s/ /T/, $timestamp .= 'Z'; $self->identifier( new HTTP::OAI::Header( identifier => $repository->{ koha_identifier} . ':' . $biblionumber, datestamp => $timestamp, ) ); $pos++; - } - $self->resumptionToken( new C4::OAI::ResumptionToken( - metadataPrefix => $token->{metadata_prefix}, - from => $token->{from}, - until => $token->{until}, - offset => $pos ) ) if ($pos > $token->{offset}); + } + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + from => $token->{from}, + until => $token->{until}, + offset => $pos, + set => $token->{set} + ) + ) if ($pos > $token->{offset}); return $self; } # __END__ C4::OAI::ListIdentifiers +package C4::OAI::Description; + +use strict; +use warnings; +use HTTP::OAI; +use HTTP::OAI::SAXHandler qw/ :SAX /; + +sub new { + my ( $class, %args ) = @_; + + my $self = {}; + + if(my $setDescription = $args{setDescription}) { + $self->{setDescription} = $setDescription; + } + if(my $handler = $args{handler}) { + $self->{handler} = $handler; + } + + bless $self, $class; + return $self; +} + +sub set_handler { + my ( $self, $handler ) = @_; + + $self->{handler} = $handler if $handler; + + return $self; +} + +sub generate { + my ( $self ) = @_; + + g_data_element($self->{handler}, 'http://www.openarchives.org/OAI/2.0/', 'setDescription', {}, $self->{setDescription}); + + return $self; +} + +# __END__ C4::OAI::Description + +package C4::OAI::ListSets; + +use strict; +use warnings; +use HTTP::OAI; +use C4::OAI::Sets; + +use base ("HTTP::OAI::ListSets"); + +sub new { + my ( $class, $repository, %args ) = @_; + + my $self = HTTP::OAI::ListSets->new(%args); + + my $token = C4::OAI::ResumptionToken->new(%args); + my $sets = GetOAISets; + my $pos = 0; + foreach my $set (@$sets) { + if ($pos < $token->{offset}) { + $pos++; + next; + } + my @descriptions; + foreach my $desc (@{$set->{'descriptions'}}) { + push @descriptions, C4::OAI::Description->new( + setDescription => $desc, + ); + } + $self->set( + HTTP::OAI::Set->new( + setSpec => $set->{'spec'}, + setName => $set->{'name'}, + setDescription => \@descriptions, + ) + ); + $pos++; + last if ($pos + 1 - $token->{offset}) > $repository->{koha_max_count}; + } + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + offset => $pos + ) + ) if ( $pos > $token->{offset} ); + + return $self; +} + +# __END__ C4::OAI::ListSets; package C4::OAI::ListRecords; use strict; use warnings; -use diagnostics; use HTTP::OAI; +use C4::OAI::Sets; use base ("HTTP::OAI::ListRecords"); @@ -329,28 +445,50 @@ sub new { my $token = new C4::OAI::ResumptionToken( %args ); my $dbh = C4::Context->dbh; - my $sql = "SELECT biblionumber, marcxml, timestamp - FROM biblioitems - WHERE timestamp >= ? AND timestamp <= ? - LIMIT " . $repository->{koha_max_count} . " - OFFSET " . $token->{offset}; + my $set; + if(defined $token->{'set'}) { + $set = GetOAISetBySpec($token->{'set'}); + } + my $sql = " + SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp + FROM biblioitems + "; + $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; + $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; + $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; + $sql .= " + LIMIT $repository->{'koha_max_count'} + OFFSET $token->{'offset'} + "; + my $sth = $dbh->prepare( $sql ); - $sth->execute( $token->{from}, $token->{until} ); + my @bind_params = ($token->{'from'}, $token->{'until'}); + push @bind_params, $set->{'id'} if defined $set; + $sth->execute( @bind_params ); my $pos = $token->{offset}; - while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) { + while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) { + my $oai_sets = GetOAISetsBiblio($biblionumber); + my @setSpecs; + foreach (@$oai_sets) { + push @setSpecs, $_->{spec}; + } $self->record( C4::OAI::Record->new( - $repository, $marcxml, $timestamp, + $repository, $marcxml, $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber, metadataPrefix => $token->{metadata_prefix} ) ); $pos++; - } - $self->resumptionToken( new C4::OAI::ResumptionToken( - metadataPrefix => $token->{metadata_prefix}, - from => $token->{from}, - until => $token->{until}, - offset => $pos ) ) if ($pos > $token->{offset}); + } + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + from => $token->{from}, + until => $token->{until}, + offset => $pos, + set => $token->{set} + ) + ) if ($pos > $token->{offset}); return $self; } @@ -365,7 +503,6 @@ use base ("HTTP::OAI::Repository"); use strict; use warnings; -use diagnostics; use HTTP::OAI; use HTTP::OAI::Repository qw/:validate/; @@ -418,14 +555,8 @@ sub new { else { my %attr = CGI::Vars(); my $verb = delete( $attr{verb} ); - if ( grep { $_ eq $verb } qw( ListSets ) ) { - $response = HTTP::OAI::Response->new( - requestURL => $self->self_url(), - errors => [ new HTTP::OAI::Error( - code => 'noSetHierarchy', - message => "Koha repository doesn't have sets", - ) ] , - ); + if ( $verb eq 'ListSets' ) { + $response = C4::OAI::ListSets->new($self, %attr); } elsif ( $verb eq 'Identify' ) { $response = C4::OAI::Identify->new( $self ); -- 1.7.5.4