From 9d86110e619d51b1c0a51fd14fefad8c81ef1bd6 Mon Sep 17 00:00:00 2001 From: Julian Maurice Date: Mon, 23 Jan 2012 16:25:51 +0100 Subject: [PATCH] Bug 6440: Implement OAI-PMH Sets New sql tables: - oai_sets: contains the list of sets, described by a spec and a name - oai_sets_descriptions: contains a list of descriptions for each set - oai_sets_mappings: conditions on marc fields to match for biblio to be in a set - oai_sets_biblios: list of biblionumbers for each set New admin page: allow to configure sets: - Creation, deletion, modification of spec, name and descriptions - Define mappings which will be used for building oai sets Implements OAI Sets in opac/oai.pl: - ListSets, ListIdentifiers, ListRecords, GetRecord New script misc/migration_tools/build_oai_sets.pl: - Retrieve marcxml from all biblios and test if they belong to defined sets. The oai_sets_biblios table is then updated accordingly Use OPACBaseURL in oai_dc xslt --- C4/OAI/Sets.pm | 485 ++++++++++++++++++++ admin/oai_set_mappings.pl | 86 ++++ admin/oai_sets.pl | 102 ++++ installer/data/mysql/atomicupdate/oai_sets.sql | 32 ++ installer/data/mysql/kohastructure.sql | 49 ++ installer/data/mysql/updatedatabase.pl | 10 + .../intranet-tmpl/prog/en/includes/admin-menu.inc | 1 + .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/oai_set_mappings.tt | 103 +++++ .../prog/en/modules/admin/oai_sets.tt | 140 ++++++ .../prog/en/xslt/UNIMARCslim2OAIDC.xsl | 11 +- misc/migration_tools/build_oai_sets.pl | 106 +++++ opac/oai.pl | 233 ++++++++-- 13 files changed, 1304 insertions(+), 56 deletions(-) create mode 100644 C4/OAI/Sets.pm create mode 100755 admin/oai_set_mappings.pl create mode 100755 admin/oai_sets.pl create mode 100644 installer/data/mysql/atomicupdate/oai_sets.sql create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt create mode 100755 misc/migration_tools/build_oai_sets.pl diff --git a/C4/OAI/Sets.pm b/C4/OAI/Sets.pm new file mode 100644 index 0000000..2a52a51 --- /dev/null +++ b/C4/OAI/Sets.pm @@ -0,0 +1,485 @@ +package C4::OAI::Sets; + +# Copyright 2011 BibLibre +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +C4::OAI::Sets - OAI Sets management functions + +=head1 DESCRIPTION + +C4::OAI::Sets contains functions for managing storage and editing of OAI Sets. + +OAI Set description can be found L + +=cut + +use Modern::Perl; +use C4::Context; + +use vars qw(@ISA @EXPORT); + +BEGIN { + require Exporter; + @ISA = qw(Exporter); + @EXPORT = qw( + &GetOAISets &GetOAISet &GetOAISetBySpec &ModOAISet &DelOAISet &AddOAISet + &GetOAISetsMappings &GetOAISetMappings &ModOAISetMappings + &GetOAISetsBiblio &ModOAISetsBiblios + ); +} + +=head1 FUNCTIONS + +=head2 GetOAISets + + $oai_sets = GetOAISets; + +GetOAISets return a array reference of hash references describing the sets. +The hash references looks like this: + + { + 'name' => 'set name', + 'spec' => 'set spec', + 'descriptions' => [ + 'description 1', + 'description 2', + ... + ] + } + +=cut + +sub GetOAISets { + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * FROM oai_sets + }; + my $sth = $dbh->prepare($query); + $sth->execute; + my $results = $sth->fetchall_arrayref({}); + + $query = qq{ + SELECT description + FROM oai_sets_descriptions + WHERE set_id = ? + }; + $sth = $dbh->prepare($query); + foreach my $set (@$results) { + $sth->execute($set->{'id'}); + my $desc = $sth->fetchall_arrayref({}); + foreach (@$desc) { + push @{$set->{'descriptions'}}, $_->{'description'}; + } + } + + return $results; +} + +=head2 GetOAISet + + $set = GetOAISet($set_id); + +GetOAISet returns a hash reference describing the set with the given set_id. + +See GetOAISets to see what the hash looks like. + +=cut + +sub GetOAISet { + my ($set_id) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * + FROM oai_sets + WHERE id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); + my $set = $sth->fetchrow_hashref; + + $query = qq{ + SELECT description + FROM oai_sets_descriptions + WHERE set_id = ? + }; + $sth = $dbh->prepare($query); + $sth->execute($set->{'id'}); + my $desc = $sth->fetchall_arrayref({}); + foreach (@$desc) { + push @{$set->{'descriptions'}}, $_->{'description'}; + } + + return $set; +} + +=head2 GetOAISetBySpec + + my $set = GetOAISetBySpec($setSpec); + +Returns a hash describing the set whose spec is $setSpec + +=cut + +sub GetOAISetBySpec { + my $setSpec = shift; + + return unless defined $setSpec; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * + FROM oai_sets + WHERE spec = ? + LIMIT 1 + }; + my $sth = $dbh->prepare($query); + $sth->execute($setSpec); + + return $sth->fetchrow_hashref; +} + +=head2 ModOAISet + + my $set = { + 'id' => $set_id, # mandatory + 'spec' => $spec, # mandatory + 'name' => $name, # mandatory + 'descriptions => \@descriptions, # optional, [] to remove descriptions + }; + ModOAISet($set); + +ModOAISet modify a set in the database. + +=cut + +sub ModOAISet { + my ($set) = @_; + + return unless($set && $set->{'spec'} && $set->{'name'}); + + if(!defined $set->{'id'}) { + warn "Set ID not defined, can't modify the set"; + return; + } + + my $dbh = C4::Context->dbh; + my $query = qq{ + UPDATE oai_sets + SET spec = ?, + name = ? + WHERE id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set->{'spec'}, $set->{'name'}, $set->{'id'}); + + if($set->{'descriptions'}) { + $query = qq{ + DELETE FROM oai_sets_descriptions + WHERE set_id = ? + }; + $sth = $dbh->prepare($query); + $sth->execute($set->{'id'}); + + if(scalar @{$set->{'descriptions'}} > 0) { + $query = qq{ + INSERT INTO oai_sets_descriptions (set_id, description) + VALUES (?,?) + }; + $sth = $dbh->prepare($query); + foreach (@{ $set->{'descriptions'} }) { + $sth->execute($set->{'id'}, $_) if $_; + } + } + } +} + +=head2 DelOAISet + + DelOAISet($set_id); + +DelOAISet remove the set with the given set_id + +=cut + +sub DelOAISet { + my ($set_id) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + DELETE oai_sets, oai_sets_descriptions, oai_sets_mappings + FROM oai_sets + LEFT JOIN oai_sets_descriptions ON oai_sets_descriptions.set_id = oai_sets.id + LEFT JOIN oai_sets_mappings ON oai_sets_mappings.set_id = oai_sets.id + WHERE oai_sets.id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); +} + +=head2 AddOAISet + + my $set = { + 'id' => $set_id, # mandatory + 'spec' => $spec, # mandatory + 'name' => $name, # mandatory + 'descriptions => \@descriptions, # optional + }; + my $set_id = AddOAISet($set); + +AddOAISet adds a new set and returns its id, or undef if something went wrong. + +=cut + +sub AddOAISet { + my ($set) = @_; + + return unless($set && $set->{'spec'} && $set->{'name'}); + + my $set_id; + my $dbh = C4::Context->dbh; + my $query = qq{ + INSERT INTO oai_sets (spec, name) + VALUES (?,?) + }; + my $sth = $dbh->prepare($query); + if( $sth->execute($set->{'spec'}, $set->{'name'}) ) { + $set_id = $dbh->last_insert_id(undef, undef, 'oai_sets', undef); + if($set->{'descriptions'}) { + $query = qq{ + INSERT INTO oai_sets_descriptions (set_id, description) + VALUES (?,?) + }; + $sth = $dbh->prepare($query); + foreach( @{ $set->{'descriptions'} } ) { + $sth->execute($set_id, $_) if $_; + } + } + } else { + warn "AddOAISet failed"; + } + + return $set_id; +} + +=head2 GetOAISetsMappings + + my $mappings = GetOAISetsMappings; + +GetOAISetsMappings returns mappings for all OAI Sets. + +Mappings define how biblios are categorized in sets. +A mapping is defined by three properties: + + { + marcfield => 'XXX', # the MARC field to check + marcsubfield => 'Y', # the MARC subfield to check + marcvalue => 'zzzz', # the value to check + } + +If defined in a set mapping, a biblio which have at least one 'Y' subfield of +one 'XXX' field equal to 'zzzz' will belong to this set. +If multiple mappings are defined in a set, the biblio will belong to this set +if at least one condition is matched. + +GetOAISetsMappings returns a hashref of arrayrefs of hashrefs. +The first hashref keys are the sets IDs, so it looks like this: + + $mappings = { + '1' => [ + { + marcfield => 'XXX', + marcsubfield => 'Y', + marcvalue => 'zzzz' + }, + { + ... + }, + ... + ], + '2' => [...], + ... + }; + +=cut + +sub GetOAISetsMappings { + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * FROM oai_sets_mappings + }; + my $sth = $dbh->prepare($query); + $sth->execute; + + my $mappings = {}; + while(my $result = $sth->fetchrow_hashref) { + push @{ $mappings->{$result->{'set_id'}} }, { + marcfield => $result->{'marcfield'}, + marcsubfield => $result->{'marcsubfield'}, + marcvalue => $result->{'marcvalue'} + }; + } + + return $mappings; +} + +=head2 GetOAISetMappings + + my $set_mappings = GetOAISetMappings($set_id); + +Return mappings for the set with given set_id. It's an arrayref of hashrefs + +=cut + +sub GetOAISetMappings { + my ($set_id) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT * + FROM oai_sets_mappings + WHERE set_id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); + + my @mappings; + while(my $result = $sth->fetchrow_hashref) { + push @mappings, { + marcfield => $result->{'marcfield'}, + marcsubfield => $result->{'marcsubfield'}, + marcvalue => $result->{'marcvalue'} + }; + } + + return \@mappings; +} + +=head2 ModOAISetMappings { + + my $mappings = [ + { + marcfield => 'XXX', + marcsubfield => 'Y', + marcvalue => 'zzzz' + }, + ... + ]; + ModOAISetMappings($set_id, $mappings); + +ModOAISetMappings modifies mappings of a given set. + +=cut + +sub ModOAISetMappings { + my ($set_id, $mappings) = @_; + + return unless $set_id; + + my $dbh = C4::Context->dbh; + my $query = qq{ + DELETE FROM oai_sets_mappings + WHERE set_id = ? + }; + my $sth = $dbh->prepare($query); + $sth->execute($set_id); + + if(scalar @$mappings > 0) { + $query = qq{ + INSERT INTO oai_sets_mappings (set_id, marcfield, marcsubfield, marcvalue) + VALUES (?,?,?,?) + }; + $sth = $dbh->prepare($query); + foreach (@$mappings) { + $sth->execute($set_id, $_->{'marcfield'}, $_->{'marcsubfield'}, $_->{'marcvalue'}); + } + } +} + +=head2 GetOAISetsBiblio + + $oai_sets = GetOAISetsBiblio($biblionumber); + +Return the OAI sets where biblio appears. + +Return value is an arrayref of hashref where each element of the array is a set. +Keys of hash are id, spec and name + +=cut + +sub GetOAISetsBiblio { + my ($biblionumber) = @_; + + my $dbh = C4::Context->dbh; + my $query = qq{ + SELECT oai_sets.* + FROM oai_sets + LEFT JOIN oai_sets_biblios ON oai_sets_biblios.set_id = oai_sets.id + WHERE biblionumber = ? + }; + my $sth = $dbh->prepare($query); + + $sth->execute($biblionumber); + return $sth->fetchall_arrayref({}); +} + +=head2 ModOAISetsBiblios + + my $oai_sets_biblios = { + '1' => [1, 3, 4], # key is the set_id, and value is an array ref of biblionumbers + '2' => [], + ... + }; + ModOAISetsBiblios($oai_sets_biblios); + +ModOAISetsBiblios fill oai_sets_biblios table with given infos. +This table is then used in opac/oai.pl. + +=cut + +sub ModOAISetsBiblios { + my $oai_sets_biblios = shift; + + return unless ref($oai_sets_biblios) eq "HASH"; + + my $dbh = C4::Context->dbh; + my $query = qq{ + TRUNCATE TABLE oai_sets_biblios + }; + my $sth = $dbh->prepare($query); + $sth->execute; + $query = qq{ + INSERT INTO oai_sets_biblios (set_id, biblionumber) + VALUES (?,?) + }; + $sth = $dbh->prepare($query); + foreach my $set_id (keys %$oai_sets_biblios) { + foreach my $biblionumber (@{$oai_sets_biblios->{$set_id}}) { + $sth->execute($set_id, $biblionumber); + } + } +} + +1; diff --git a/admin/oai_set_mappings.pl b/admin/oai_set_mappings.pl new file mode 100755 index 0000000..4d570f9 --- /dev/null +++ b/admin/oai_set_mappings.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +# Copyright 2011 BibLibre SARL +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +oai_set_mappings.pl + +=head1 DESCRIPTION + +Define mappings for a given set. +Mappings are conditions that define which biblio is included in which set. +A condition is in the form 200$a = 'abc'. +Multiple conditions can be defined for a given set. In this case, +the OR operator will be applied. + +=cut + +use Modern::Perl; + +use CGI; +use C4::Auth; +use C4::Output; +use C4::OAI::Sets; + +use Data::Dumper; + +my $input = new CGI; +my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( { + template_name => 'admin/oai_set_mappings.tt', + query => $input, + type => 'intranet', + authnotrequired => 0, + flagsrequired => { 'parameters' => '*' }, + debug => 1, +} ); + +my $id = $input->param('id'); +my $op = $input->param('op'); + +if($op && $op eq "save") { + my @marcfields = $input->param('marcfield'); + my @marcsubfields = $input->param('marcsubfield'); + my @marcvalues = $input->param('marcvalue'); + + my @mappings; + my $i = 0; + while($i < @marcfields and $i < @marcsubfields and $i < @marcvalues) { + if($marcfields[$i] and $marcsubfields[$i] and $marcvalues[$i]) { + push @mappings, { + marcfield => $marcfields[$i], + marcsubfield => $marcsubfields[$i], + marcvalue => $marcvalues[$i] + }; + } + $i++; + } + ModOAISetMappings($id, \@mappings); + $template->param(mappings_saved => 1); +} + +my $set = GetOAISet($id); +my $mappings = GetOAISetMappings($id); + +$template->param( + id => $id, + setName => $set->{'name'}, + setSpec => $set->{'spec'}, + mappings => $mappings, +); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/admin/oai_sets.pl b/admin/oai_sets.pl new file mode 100755 index 0000000..a826107 --- /dev/null +++ b/admin/oai_sets.pl @@ -0,0 +1,102 @@ +#!/usr/bin/perl + +# Copyright 2011 BibLibre SARL +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +oai_sets.pl + +=head1 DESCRIPTION + +Admin page to describe OAI SETs + +=cut + +use Modern::Perl; + +use CGI; +use C4::Auth; +use C4::Output; +use C4::OAI::Sets; + +use Data::Dumper; + +my $input = new CGI; +my ($template, $loggedinuser, $cookie, $flags) = get_template_and_user( { + template_name => 'admin/oai_sets.tt', + query => $input, + type => 'intranet', + authnotrequired => 0, + flagsrequired => { 'parameters' => '*' }, + debug => 1, +} ); + +my $op = $input->param('op'); + +if($op && $op eq "new") { + $template->param( op_new => 1 ); +} elsif($op && $op eq "savenew") { + my $spec = $input->param('spec'); + my $name = $input->param('name'); + my @descriptions = $input->param('description'); + AddOAISet({ + spec => $spec, + name => $name, + descriptions => \@descriptions + }); +} elsif($op && $op eq "mod") { + my $id = $input->param('id'); + my $set = GetOAISet($id); + $template->param( + op_mod => 1, + id => $set->{'id'}, + spec => $set->{'spec'}, + name => $set->{'name'}, + descriptions => [ map { {description => $_} } @{ $set->{'descriptions'} } ], + ); +} elsif($op && $op eq "savemod") { + my $id = $input->param('id'); + my $spec = $input->param('spec'); + my $name = $input->param('name'); + my @descriptions = $input->param('description'); + ModOAISet({ + id => $id, + spec => $spec, + name => $name, + descriptions => \@descriptions + }); +} elsif($op && $op eq "del") { + my $id = $input->param('id'); + DelOAISet($id); +} + +my $OAISets = GetOAISets; +my @sets_loop; +foreach(@$OAISets) { + push @sets_loop, { + id => $_->{'id'}, + spec => $_->{'spec'}, + name => $_->{'name'}, + descriptions => [ map { {description => $_} } @{ $_->{'descriptions'} } ] + }; +} + +$template->param( + sets_loop => \@sets_loop, +); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/installer/data/mysql/atomicupdate/oai_sets.sql b/installer/data/mysql/atomicupdate/oai_sets.sql new file mode 100644 index 0000000..d5bdb64 --- /dev/null +++ b/installer/data/mysql/atomicupdate/oai_sets.sql @@ -0,0 +1,32 @@ +DROP TABLE IF EXISTS `oai_sets`; +CREATE TABLE `oai_sets` ( + `id` int(11) NOT NULL auto_increment, + `spec` varchar(80) NOT NULL UNIQUE, + `name` varchar(80) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `oai_sets_descriptions`; +CREATE TABLE `oai_sets_descriptions` ( + `set_id` int(11) NOT NULL, + `description` varchar(255) NOT NULL, + CONSTRAINT `oai_sets_descriptions_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `oai_sets_mappings`; +CREATE TABLE `oai_sets_mappings` ( + `set_id` int(11) NOT NULL, + `marcfield` char(3) NOT NULL, + `marcsubfield` char(1) NOT NULL, + `marcvalue` varchar(80) NOT NULL, + CONSTRAINT `oai_sets_mappings_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `oai_sets_biblios`; +CREATE TABLE `oai_sets_biblios` ( + `biblionumber` int(11) NOT NULL, + `set_id` int(11) NOT NULL, + PRIMARY KEY (`biblionumber`, `set_id`), + CONSTRAINT `oai_sets_biblios_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `oai_sets_biblios_ibfk_2` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 5673e35..2379615 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -1353,6 +1353,55 @@ CREATE TABLE `nozebra` ( ENGINE=InnoDB DEFAULT CHARSET=utf8; -- +-- Table structure for table `oai_sets` +-- + +DROP TABLE IF EXISTS `oai_sets`; +CREATE TABLE `oai_sets` ( + `id` int(11) NOT NULL auto_increment, + `spec` varchar(80) NOT NULL UNIQUE, + `name` varchar(80) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_sets_descriptions` +-- + +DROP TABLE IF EXISTS `oai_sets_descriptions`; +CREATE TABLE `oai_sets_descriptions` ( + `set_id` int(11) NOT NULL, + `description` varchar(255) NOT NULL, + CONSTRAINT `oai_sets_descriptions_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_sets_mappings` +-- + +DROP TABLE IF EXISTS `oai_sets_mappings`; +CREATE TABLE `oai_sets_mappings` ( + `set_id` int(11) NOT NULL, + `marcfield` char(3) NOT NULL, + `marcsubfield` char(1) NOT NULL, + `marcvalue` varchar(80) NOT NULL, + CONSTRAINT `oai_sets_mappings_ibfk_1` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- +-- Table structure for table `oai_sets_biblios` +-- + +DROP TABLE IF EXISTS `oai_sets_biblios`; +CREATE TABLE `oai_sets_biblios` ( + `biblionumber` int(11) NOT NULL, + `set_id` int(11) NOT NULL, + PRIMARY KEY (`biblionumber`, `set_id`), + CONSTRAINT `oai_sets_biblios_ibfk_1` FOREIGN KEY (`biblionumber`) REFERENCES `biblio` (`biblionumber`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `oai_sets_biblios_ibfk_2` FOREIGN KEY (`set_id`) REFERENCES `oai_sets` (`id`) ON DELETE CASCADE ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- -- Table structure for table `old_issues` -- diff --git a/installer/data/mysql/updatedatabase.pl b/installer/data/mysql/updatedatabase.pl index b12196c..7537fea 100755 --- a/installer/data/mysql/updatedatabase.pl +++ b/installer/data/mysql/updatedatabase.pl @@ -4626,6 +4626,16 @@ if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { SetVersion($DBversion); } +$DBversion = "XXX"; +if ( C4::Context->preference("Version") < TransformToNum($DBversion) ) { + my $installer = C4::Installer->new(); + my $full_path = C4::Context->config('intranetdir') . "/installer/data/$installer->{dbms}/atomicupdate/oai_sets.sql"; + my $error = $installer->load_sql($full_path); + warn $error if $error; + print "Upgrade to $DBversion done (Add OAI SETs tables)\n"; + SetVersion($DBversion); +} + =head1 FUNCTIONS =head2 DropAllForeignKeys($table) diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index e9537f4..74dd667 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -43,6 +43,7 @@
  • Authority types
  • Classification sources
  • Record matching rules
    • +
  • OAI Sets configuration
    • Acquisition parameters
    diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index fe76ae8..cea2215 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -74,6 +74,8 @@
    Define classification sources (i.e., call number schemes) used by your collection. Also define filing rules used for sorting call numbers.
    Record matching rules
    Manage rules for automatically matching MARC records during record imports.
    +
    OAI Sets Configuration
    +
    Manage OAI Sets

    Acquisition parameters

    diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt new file mode 100644 index 0000000..894d7d5 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_set_mappings.tt @@ -0,0 +1,103 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Admin › OAI Set Mappings +[% INCLUDE 'doc-head-close.inc' %] + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
    + +
    +
    +
    + [% IF ( mappings_saved ) %] +
    +

    Mappings have been saved

    +

    Return to sets management

    +
    + [% END %] +

    Mappings for set '[% setName %]' ([% setSpec %])

    + [% UNLESS ( mappings ) %] +

    Warning: no mappings defined for this set

    + [% END %] +
    + + + + + + + + + + + + [% IF ( mappings ) %] + [% FOREACH mapping IN mappings %] + + + + + + + + [% END %] + [% ELSE %] + + + + + + + + [% END %] + +
    FieldSubfield Value 
    is equal to + [% IF ( loop.last ) %] + + [% ELSE %] + OR + [% END %] +
    is equal to
    +

    Hint: to delete a line, empty at least one of the text fields in this line

    + + +
    + + +
    +
    + +
    +
    +
    + [% INCLUDE 'admin-menu.inc' %] +
    +
    +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt new file mode 100644 index 0000000..2cb3e57 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/oai_sets.tt @@ -0,0 +1,140 @@ +[% INCLUDE 'doc-head-open.inc' %] +Koha › Admin › OAI Sets +[% INCLUDE 'doc-head-close.inc' %] + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
    + +
    +
    +
    +

    OAI Sets Configuration

    + + [% IF op_new %] +

    Add a new set

    +
    + +
    + + +
    + + +
    + +
      +
    + Add description +
    + + +
    + [% ELSE %][% IF op_mod %] +

    Modify set '[% spec %]'

    +
    + + +
    + + +
    + + +
    + +
      + [% FOREACH desc IN descriptions %] +
    • + +  × +
      • + [% END %] +
    + Add description +
    + + +
    + [% END %] + [% END %] + +

    List of sets

    + [% UNLESS ( op_new ) %] + Add a new set + [% END %] + [% IF sets_loop %] + + + + + + + + + + + [% FOREACH set IN sets_loop %] + + + + + + + [% END %] + +
    setSpecsetNamesetDescriptionsAction
    [% set.spec %][% set.name %] + [% IF set.descriptions %] +
      + [% FOREACH desc IN set.descriptions %] +
    • [% desc.description %]
      • + [% END %] +
    + [% ELSE %] + No descriptions + [% END %] +     		+ Modify + | + Delete + | + Define mappings +
    + [% ELSE %] +

    There is no set defined.

    + [% END %] + + +
    +
    +
    + [% INCLUDE 'admin-menu.inc' %] +
    +
    +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl b/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl index 6352384..a75b800 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl +++ b/koha-tmpl/intranet-tmpl/prog/en/xslt/UNIMARCslim2OAIDC.xsl @@ -163,9 +163,10 @@ - - http://opac.mylibrary.org/bib/ - + + + /bib/ + @@ -175,10 +176,10 @@ Main Branch Library 2 - + : - + diff --git a/misc/migration_tools/build_oai_sets.pl b/misc/migration_tools/build_oai_sets.pl new file mode 100755 index 0000000..bc33255 --- /dev/null +++ b/misc/migration_tools/build_oai_sets.pl @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +# Turn on autoflush +$| = 1; + +use Modern::Perl; +use MARC::Record; +use MARC::File::XML; +use List::MoreUtils qw/uniq/; + +use C4::Context; +use C4::Charset qw/StripNonXmlChars/; +use C4::OAI::Sets qw/GetOAISets GetOAISetBySpec GetOAISetsMappings ModOAISetsBiblios GetOAISet/; + +my $debug = $ENV{'DEBUG'}; + +my $dbh = C4::Context->dbh; + +# Get OAI sets mappings +my $mappings = GetOAISetsMappings; + +# Get all biblionumbers and marcxml +print "Retrieving all biblios... "; +my $query = qq{ + SELECT biblionumber, marcxml + FROM biblioitems +}; +my $sth = $dbh->prepare($query); +$sth->execute; +my $results = $sth->fetchall_arrayref({}); +print "Done.\n"; + +# Build lists of parents sets +my $sets = GetOAISets; +my $parentsets; +foreach my $set (@$sets) { + my $setSpec = $set->{'spec'}; + while($setSpec =~ /^(.+):(.+)$/) { + my $parent = $1; + my $parent_set = GetOAISetBySpec($parent); + if($parent_set) { + push @{ $parentsets->{$set->{'id'}} }, $parent_set->{'id'}; + $setSpec = $parent; + } else { + last; + } + } +} + +my $num_biblios = scalar @$results; +my $i = 1; +my $sets_biblios = {}; +foreach my $res (@$results) { + my $biblionumber = $res->{'biblionumber'}; + my $marcxml = $res->{'marcxml'}; + if($i % 1000 == 0) { + my $percent = ($i * 100) / $num_biblios; + $percent = sprintf("%.2f", $percent); + say "Progression: $i/$num_biblios ($percent %)"; + } + # The following lines are copied from GetMarcBiblio + # We don't call GetMarcBiblio to avoid a sql query to be executed each time + $marcxml = StripNonXmlChars($marcxml); + MARC::File::XML->default_record_format(C4::Context->preference('marcflavour')); + my $record; + eval { + $record = MARC::Record::new_from_xml($marcxml, "utf8", C4::Context->preference('marcflavour')); + }; + if($@) { + warn "(biblio $biblionumber) Error while creating record from marcxml: $@"; + next; + } + + foreach my $set_id (keys %$mappings) { + foreach my $mapping (@{ $mappings->{$set_id} }) { + next if not $mapping; + my $field = $mapping->{'marcfield'}; + my $subfield = $mapping->{'marcsubfield'}; + my $value = $mapping->{'marcvalue'}; + + my @subfield_values = $record->subfield($field, $subfield); + if(0 < grep /^$value$/, @subfield_values) { + $debug and say "Biblio $biblionumber belong to set $set_id, $field\$$subfield = $value"; + push @{ $sets_biblios->{$set_id} }, $biblionumber; + foreach my $parent_set_id ( @{ $parentsets->{$set_id} } ) { + push @{ $sets_biblios->{$parent_set_id} }, $biblionumber; + } + last; + } + } + } + $i++; +} +say "Progression: Done."; + +say "Summary:"; +foreach my $set_id (keys %$sets_biblios) { + $sets_biblios->{$set_id} = [ uniq @{ $sets_biblios->{$set_id} } ]; + my $set = GetOAISet($set_id); + my $setSpec = $set->{'spec'}; + say "Set '$setSpec': ". scalar(@{$sets_biblios->{$set_id}}) ." biblios"; +} + +print "Updating database... "; +ModOAISetsBiblios($sets_biblios); +print "Done.\n"; diff --git a/opac/oai.pl b/opac/oai.pl index 038b6d1..72a722e 100755 --- a/opac/oai.pl +++ b/opac/oai.pl @@ -2,7 +2,6 @@ use strict; use warnings; -use diagnostics; use CGI qw/:standard -oldstyle_urls/; use vars qw( $GZIP ); @@ -59,7 +58,6 @@ package C4::OAI::ResumptionToken; use strict; use warnings; -use diagnostics; use HTTP::OAI; use base ("HTTP::OAI::ResumptionToken"); @@ -70,9 +68,9 @@ sub new { my $self = $class->SUPER::new(%args); - my ($metadata_prefix, $offset, $from, $until); + my ($metadata_prefix, $offset, $from, $until, $set); if ( $args{ resumptionToken } ) { - ($metadata_prefix, $offset, $from, $until) + ($metadata_prefix, $offset, $from, $until, $set) = split( ':', $args{resumptionToken} ); } else { @@ -84,15 +82,17 @@ sub new { $until = sprintf( "%.4d-%.2d-%.2d", $year+1900, $mon+1,$mday ); } $offset = $args{ offset } || 0; + $set = $args{set}; } $self->{ metadata_prefix } = $metadata_prefix; $self->{ offset } = $offset; $self->{ from } = $from; $self->{ until } = $until; + $self->{ set } = $set; $self->resumptionToken( - join( ':', $metadata_prefix, $offset, $from, $until ) ); + join( ':', $metadata_prefix, $offset, $from, $until, $set ) ); $self->cursor( $offset ); return $self; @@ -106,7 +106,6 @@ package C4::OAI::Identify; use strict; use warnings; -use diagnostics; use HTTP::OAI; use C4::Context; @@ -145,7 +144,6 @@ package C4::OAI::ListMetadataFormats; use strict; use warnings; -use diagnostics; use HTTP::OAI; use base ("HTTP::OAI::ListMetadataFormats"); @@ -188,14 +186,13 @@ package C4::OAI::Record; use strict; use warnings; -use diagnostics; use HTTP::OAI; use HTTP::OAI::Metadata::OAI_DC; use base ("HTTP::OAI::Record"); sub new { - my ($class, $repository, $marcxml, $timestamp, %args) = @_; + my ($class, $repository, $marcxml, $timestamp, $setSpecs, %args) = @_; my $self = $class->SUPER::new(%args); @@ -205,11 +202,18 @@ sub new { datestamp => $timestamp, ) ); + foreach my $setSpec (@$setSpecs) { + $self->header->setSpec($setSpec); + } + my $parser = XML::LibXML->new(); my $record_dom = $parser->parse_string( $marcxml ); my $format = $args{metadataPrefix}; if ( $format ne 'marcxml' ) { - $record_dom = $repository->stylesheet($format)->transform( $record_dom ); + my %args = ( + OPACBaseURL => "'" . C4::Context->preference('OPACBaseURL') . "'" + ); + $record_dom = $repository->stylesheet($format)->transform($record_dom, %args); } $self->metadata( HTTP::OAI::Metadata->new( dom => $record_dom ) ); @@ -224,8 +228,8 @@ package C4::OAI::GetRecord; use strict; use warnings; -use diagnostics; use HTTP::OAI; +use C4::OAI::Sets; use base ("HTTP::OAI::GetRecord"); @@ -254,9 +258,15 @@ sub new { ); } + my $oai_sets = GetOAISetsBiblio($biblionumber); + my @setSpecs; + foreach (@$oai_sets) { + push @setSpecs, $_->{spec}; + } + #$self->header( HTTP::OAI::Header->new( identifier => $args{identifier} ) ); $self->record( C4::OAI::Record->new( - $repository, $marcxml, $timestamp, %args ) ); + $repository, $marcxml, $timestamp, \@setSpecs, %args ) ); return $self; } @@ -269,8 +279,8 @@ package C4::OAI::ListIdentifiers; use strict; use warnings; -use diagnostics; use HTTP::OAI; +use C4::OAI::Sets; use base ("HTTP::OAI::ListIdentifiers"); @@ -282,42 +292,148 @@ sub new { my $token = new C4::OAI::ResumptionToken( %args ); my $dbh = C4::Context->dbh; - my $sql = "SELECT biblionumber, timestamp - FROM biblioitems - WHERE timestamp >= ? AND timestamp <= ? - LIMIT " . $repository->{koha_max_count} . " - OFFSET " . $token->{offset}; + my $set; + if(defined $token->{'set'}) { + $set = GetOAISetBySpec($token->{'set'}); + } + my $sql = " + SELECT biblioitems.biblionumber, biblioitems.timestamp + FROM biblioitems + "; + $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; + $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; + $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; + $sql .= " + LIMIT $repository->{'koha_max_count'} + OFFSET $token->{'offset'} + "; my $sth = $dbh->prepare( $sql ); - $sth->execute( $token->{from}, $token->{until} ); + my @bind_params = ($token->{'from'}, $token->{'until'}); + push @bind_params, $set->{'id'} if defined $set; + $sth->execute( @bind_params ); my $pos = $token->{offset}; - while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { - $timestamp =~ s/ /T/, $timestamp .= 'Z'; + while ( my ($biblionumber, $timestamp) = $sth->fetchrow ) { + $timestamp =~ s/ /T/, $timestamp .= 'Z'; $self->identifier( new HTTP::OAI::Header( identifier => $repository->{ koha_identifier} . ':' . $biblionumber, datestamp => $timestamp, ) ); $pos++; - } - $self->resumptionToken( new C4::OAI::ResumptionToken( - metadataPrefix => $token->{metadata_prefix}, - from => $token->{from}, - until => $token->{until}, - offset => $pos ) ) if ($pos > $token->{offset}); + } + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + from => $token->{from}, + until => $token->{until}, + offset => $pos, + set => $token->{set} + ) + ) if ($pos > $token->{offset}); return $self; } # __END__ C4::OAI::ListIdentifiers +package C4::OAI::Description; + +use strict; +use warnings; +use HTTP::OAI; +use HTTP::OAI::SAXHandler qw/ :SAX /; + +sub new { + my ( $class, %args ) = @_; + + my $self = {}; + + if(my $setDescription = $args{setDescription}) { + $self->{setDescription} = $setDescription; + } + if(my $handler = $args{handler}) { + $self->{handler} = $handler; + } + + bless $self, $class; + return $self; +} + +sub set_handler { + my ( $self, $handler ) = @_; + + $self->{handler} = $handler if $handler; + + return $self; +} + +sub generate { + my ( $self ) = @_; + + g_data_element($self->{handler}, 'http://www.openarchives.org/OAI/2.0/', 'setDescription', {}, $self->{setDescription}); + + return $self; +} + +# __END__ C4::OAI::Description + +package C4::OAI::ListSets; + +use strict; +use warnings; +use HTTP::OAI; +use C4::OAI::Sets; + +use base ("HTTP::OAI::ListSets"); + +sub new { + my ( $class, $repository, %args ) = @_; + + my $self = HTTP::OAI::ListSets->new(%args); + + my $token = C4::OAI::ResumptionToken->new(%args); + my $sets = GetOAISets; + my $pos = 0; + foreach my $set (@$sets) { + if ($pos < $token->{offset}) { + $pos++; + next; + } + my @descriptions; + foreach my $desc (@{$set->{'descriptions'}}) { + push @descriptions, C4::OAI::Description->new( + setDescription => $desc, + ); + } + $self->set( + HTTP::OAI::Set->new( + setSpec => $set->{'spec'}, + setName => $set->{'name'}, + setDescription => \@descriptions, + ) + ); + $pos++; + last if ($pos + 1 - $token->{offset}) > $repository->{koha_max_count}; + } + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + offset => $pos + ) + ) if ( $pos > $token->{offset} ); + + return $self; +} + +# __END__ C4::OAI::ListSets; package C4::OAI::ListRecords; use strict; use warnings; -use diagnostics; use HTTP::OAI; +use C4::OAI::Sets; use base ("HTTP::OAI::ListRecords"); @@ -329,28 +445,50 @@ sub new { my $token = new C4::OAI::ResumptionToken( %args ); my $dbh = C4::Context->dbh; - my $sql = "SELECT biblionumber, marcxml, timestamp - FROM biblioitems - WHERE timestamp >= ? AND timestamp <= ? - LIMIT " . $repository->{koha_max_count} . " - OFFSET " . $token->{offset}; + my $set; + if(defined $token->{'set'}) { + $set = GetOAISetBySpec($token->{'set'}); + } + my $sql = " + SELECT biblioitems.biblionumber, biblioitems.marcxml, biblioitems.timestamp + FROM biblioitems + "; + $sql .= " JOIN oai_sets_biblios ON biblioitems.biblionumber = oai_sets_biblios.biblionumber " if defined $set; + $sql .= " WHERE DATE(timestamp) >= ? AND DATE(timestamp) <= ? "; + $sql .= " AND oai_sets_biblios.set_id = ? " if defined $set; + $sql .= " + LIMIT $repository->{'koha_max_count'} + OFFSET $token->{'offset'} + "; + my $sth = $dbh->prepare( $sql ); - $sth->execute( $token->{from}, $token->{until} ); + my @bind_params = ($token->{'from'}, $token->{'until'}); + push @bind_params, $set->{'id'} if defined $set; + $sth->execute( @bind_params ); my $pos = $token->{offset}; - while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) { + while ( my ($biblionumber, $marcxml, $timestamp) = $sth->fetchrow ) { + my $oai_sets = GetOAISetsBiblio($biblionumber); + my @setSpecs; + foreach (@$oai_sets) { + push @setSpecs, $_->{spec}; + } $self->record( C4::OAI::Record->new( - $repository, $marcxml, $timestamp, + $repository, $marcxml, $timestamp, \@setSpecs, identifier => $repository->{ koha_identifier } . ':' . $biblionumber, metadataPrefix => $token->{metadata_prefix} ) ); $pos++; - } - $self->resumptionToken( new C4::OAI::ResumptionToken( - metadataPrefix => $token->{metadata_prefix}, - from => $token->{from}, - until => $token->{until}, - offset => $pos ) ) if ($pos > $token->{offset}); + } + $self->resumptionToken( + new C4::OAI::ResumptionToken( + metadataPrefix => $token->{metadata_prefix}, + from => $token->{from}, + until => $token->{until}, + offset => $pos, + set => $token->{set} + ) + ) if ($pos > $token->{offset}); return $self; } @@ -365,7 +503,6 @@ use base ("HTTP::OAI::Repository"); use strict; use warnings; -use diagnostics; use HTTP::OAI; use HTTP::OAI::Repository qw/:validate/; @@ -418,14 +555,8 @@ sub new { else { my %attr = CGI::Vars(); my $verb = delete( $attr{verb} ); - if ( grep { $_ eq $verb } qw( ListSets ) ) { - $response = HTTP::OAI::Response->new( - requestURL => $self->self_url(), - errors => [ new HTTP::OAI::Error( - code => 'noSetHierarchy', - message => "Koha repository doesn't have sets", - ) ] , - ); + if ( $verb eq 'ListSets' ) { + $response = C4::OAI::ListSets->new($self, %attr); } elsif ( $verb eq 'Identify' ) { $response = C4::OAI::Identify->new( $self ); -- 1.7.8.3