@@ -, +, @@ --- Koha/OAI/Harvester/Import/RDFXML.pm | 124 ++++++++++++++++++++- Koha/OAI/Harvester/Import/Record.pm | 113 +++++++++++++++++-- Koha/OAI/Harvester/Request.pm | 19 ++++ Koha/OAI/Harvester/Worker/Download/Stream.pm | 2 + Koha/OAI/Harvester/Worker/Import.pm | 2 + Koha/Schema/Result/OaiHarvesterRequest.pm | 12 +- installer/data/mysql/atomicupdate/bug_10662.sql | 1 + installer/data/mysql/kohastructure.sql | 1 + .../modules/tools/oai-pmh-harvester/dashboard.tt | 3 + .../en/modules/tools/oai-pmh-harvester/request.tt | 6 + t/db_dependent/OAI/Harvester/Importer/rdf.t | 53 +++++++++ tools/oai-pmh-harvester/dashboard.pl | 3 +- tools/oai-pmh-harvester/request.pl | 2 + 13 files changed, 321 insertions(+), 20 deletions(-) create mode 100644 t/db_dependent/OAI/Harvester/Importer/rdf.t --- a/Koha/OAI/Harvester/Import/RDFXML.pm +++ a/Koha/OAI/Harvester/Import/RDFXML.pm @@ -19,6 +19,12 @@ package Koha::OAI::Harvester::Import::RDFXML; # use Modern::Perl; +use Encode; +use RDF::Trine; +use RDF::Query; + +use C4::Context; +use Koha::RDF; sub new { my ($class, $args) = @_; @@ -26,21 +32,127 @@ sub new { return bless ($args, $class); } +sub _parse_rdf_into_model { + my ($self, $args) = @_; + + #Create a RDFXML parser + my $parser = RDF::Trine::Parser->new('rdfxml'); + + #Create a model in memory into which we can parse the incoming triples for processing + my $store = RDF::Trine::Store::Memory->new(); + my $model = RDF::Trine::Model->new($store); + + my $dom = $self->{dom}; + + my $byte_string = $dom->toString(2); + + #Change a byte string to a character string + my $character_string = Encode::decode('UTF-8', $byte_string, Encode::FB_CROAK); + #NOTE: RDF::Trine::Parser::RDFXML expects characters and converts them to bytes, so + #we have to change to characters before parsing. + + #FIXME: Should you provide a base uri? Unfortunately, there's no way to do this reliably over OAI-PMH. + my $base_uri = undef; + + #NOTE: UTF-8 encoding problems have been found when using XML::SAX::PurePerl. Make sure + #that you're using a parser like XML::LibXML::SAX::Parser instead. You can verify using + #sax_parser_print.pl. + $parser->parse_into_model($base_uri, $character_string, $model); + + return $model; +} + +sub _get_primary_subject { + my ($self,$args) = @_; + + my $primary_subject; + my $model = $args->{model}; + my $type = $args->{type}; + + my $object = RDF::Trine::Node::Resource->new($type); + my $object_string = $object ? $object->as_string : '?object'; + + #NOTE: Select only a subject with a particular rdf:type, and filter out + #any matches which are also objects in the same model. This is essential + #for weeding out triples that share the same rdf:type value but are linked + #objects rather than the primary subject. + my $sparql = qq~ + SELECT DISTINCT ?subject + WHERE { + ?subject $object_string . + FILTER NOT EXISTS { ?os ?op ?subject } + } + ~; + my $query = RDF::Query->new($sparql); + my $iterator = $query->execute($model); + if ($iterator){ + my @all = $iterator->get_all; + if (scalar @all == 1 ){ + my $primary = shift @all; + $primary_subject = $primary->{subject}; + } + } + return $primary_subject; +} + sub import_record { my ($self,$args) = @_; - my $framework = $args->{framework}; my $record_type = $args->{record_type}; - my $matcher = $args->{matcher}; my $koha_id = $args->{koha_id}; + my $rdf_type = $args->{rdf_type}; + my $oai_pmh_repository = $args->{oai_pmh_repository}; + my $oai_pmh_identifier = $args->{oai_pmh_identifier}; my $action = "error"; - if (! $koha_id && $matcher){ - #Create minimal MARCXML record from RDFXML to be used for Zebra-based matching. - } + #Create a model for our Koha triplestore backend + my $context = C4::Context->new(); + my $real_model = $context->triplestore('update') if $context && $context->triplestore('update'); + my $memory_model = $self->_parse_rdf_into_model(); + my $iterator = $memory_model->as_stream; + if ($iterator){ + if ($memory_model && $rdf_type){ + my $incoming_subject = $self->_get_primary_subject({ + model => $memory_model, + type => $rdf_type, + }); + + if ($incoming_subject && $incoming_subject->is_resource){ + my $rdf = Koha::RDF->new(); + if ($rdf){ + $real_model->begin_bulk_ops; + + $rdf->DelNamedGraph({ + model => $real_model, + context => $incoming_subject, + }); + $rdf->AddNamedGraph({ + model => $real_model, + context => $incoming_subject, + iterator => $iterator, + }); + #NOTE: Insert your own triples containing references to the OAI-PMH repository and the OAI-PMH identifier + if ($oai_pmh_repository){ + my $repository_predicate = RDF::Trine::Node::Resource->new('http://koha-community.org/vocab/oai-pmh/repository'); + my $repository_object = RDF::Trine::Node::Resource->new($oai_pmh_repository); + my $repository_quad = RDF::Trine::Statement::Quad->new($incoming_subject, $repository_predicate, $repository_object, $incoming_subject); + $real_model->add_statement($repository_quad); + } + if ($oai_pmh_identifier){ + my $identifier_predicate = RDF::Trine::Node::Resource->new('http://koha-community.org/vocab/oai-pmh/identifier'); + my $identifier_object = RDF::Trine::Node::Resource->new($oai_pmh_identifier); + my $identifier_quad = RDF::Trine::Statement::Quad->new($incoming_subject, $identifier_predicate, $identifier_object, $incoming_subject); + $real_model->add_statement($identifier_quad); + } - #TODO: add/update triplestore + $real_model->end_bulk_ops; + + $action = 'unknown'; #FIXME: RDF::Trine doesn't return meaningful values so we can't really say much else... + } + } + } + } return ($action,$koha_id); } --- a/Koha/OAI/Harvester/Import/Record.pm +++ a/Koha/OAI/Harvester/Import/Record.pm @@ -23,6 +23,7 @@ use XML::LibXML; use XML::LibXSLT; use URI; use File::Basename; +use RDF::Query; use C4::Context; use C4::Biblio; @@ -30,6 +31,7 @@ use C4::Biblio; use Koha::Database; use Koha::OAI::Harvester::Import::MARCXML; use Koha::OAI::Harvester::Import::RDFXML; +use Koha::RDF; my $schema = Koha::Database->new()->schema(); @@ -179,6 +181,42 @@ sub _find_koha_link { return $link_id; } + +sub _find_rdf_link { + my ($self, $args) = @_; + my ($subject,$graph); + my $repository = $self->{repository}; + my $identifier = $self->{header_identifier}; + + my $context = C4::Context->new; + my $triplestore = $context->triplestore('query'); + if ($triplestore){ + if ( URI->new($repository) && URI->new($identifier) ){ + my $sparql = qq~ + PREFIX koha-oai: + SELECT ?g ?s ?identifier ?repository + WHERE { + GRAPH ?g { + ?s koha-oai:identifier ?identifier . + ?s koha-oai:repository ?repository . + ?s koha-oai:identifier <$identifier> . + ?s koha-oai:repository <$repository> . + } + } + ~; + my $query = RDF::Query->new($sparql); + my $iterator = $query->execute($triplestore); + my @rows = $iterator->get_all; + if (scalar @rows == 1){ + my $row = $rows[0]; + $subject = $row->{s}; + $graph = $row->{g}; + } + } + } + return ($subject,$graph); +} + =head3 my ($action,$record_id) = $oai_record->import_record({ @@ -198,30 +236,57 @@ sub import_record { my $framework = $args->{framework} || ''; my $record_type = $args->{record_type} || 'biblio'; my $matcher = $args->{matcher}; + my $rdf_type = $args->{rdf_type}; #NOTE: RDF my $action = "error"; + #NOTE: RDF #Find linkage between OAI-PMH repository-identifier and Koha record id my $linked_id = $self->_find_koha_link({ record_type => $record_type, }); + #NOTE: RDF + my $context = C4::Context->new(); + #Find linkage between OAI-PMH repository-identifier and RDF records in the triplestore + my ($linked_subject, $linked_graph) = $self->_find_rdf_link if $context && $context->triplestore('query'); + if ($self->is_deleted_upstream){ - #NOTE: If a record is deleted upstream, it will not contain a metadata element + #FIXME: If a record is deleted upstream, it will not contain a metadata element, so we don't know what metadata + #format we requested. Perhaps we should be recording the metadataPrefix during download and import, so we + #can have more targeted deletes. + #NOTE: "Other records, with different metadataPrefix but the same unique identifier, may remain available for the item." + #https://www.openarchives.org/OAI/openarchivesprotocol.html#DeletedRecords + + if ($linked_graph){ + my $model = $context->triplestore('update') if $context && $context->triplestore('update'); + my $rdf = Koha::RDF->new(); + if ($rdf){ + $model->begin_bulk_ops; + $rdf->DelNamedGraph({ + model => $model, + context => $linked_graph, + }); + $model->end_bulk_ops; + } + #NOTE: We don't set action here, since RDF::Trine doesn't return a useful value, plus + #the RDF is meaningless without the Koha record mentioned below anyway. + } + if ($linked_id){ $action = $self->delete_koha_record({ record_id => $linked_id, record_type => $record_type, }); - #TODO: How to handle RDF deletions? - #NOTE: If we delete the Koha record, then we lose the link to our downloaded record, so I don't know if it matters... - #...although surely we'd want some way of cleaning up records that aren't needed anymore? I suppose that could come up with a cronjob that scans for triples that aren't linked to... } else { $action = "not_found"; #NOTE: If there's no OAI-PMH repository-identifier pair in the database, #then there's no perfect way to find a linked record to delete. } + #NOTE: RDF + #TODO: Delete named graph found via $linked_graph... + #TODO: Do we also unlink the Koha record from this OAI record? I suppose so... } else { $self->set_filter($filter); @@ -230,21 +295,47 @@ sub import_record { my $import_record = $self->filter(); if ($import_record){ + #NOTE: You can inspect the object's class here for special type handling if necessary... ($action,$linked_id) = $import_record->import_record({ + oai_pmh_repository => $self->{repository}, + oai_pmh_identifier => $self->{header_identifier}, framework => $framework, record_type => $record_type, matcher => $matcher, koha_id => $linked_id, + linked_subject => $linked_subject, #NOTE: RDF + rdf_type => $rdf_type, #NOTE: RDF }); - if ($linked_id){ - #Link Koha record ID to OAI-PMH details for this record type, - #if linkage doesn't already exist. - $self->link_koha_record({ - record_type => $record_type, - koha_id => $linked_id, - }); + if ($import_record->isa('Koha::Import::MARCXML')){ + if ($linked_id){ + #Link Koha record ID to OAI-PMH details for this record type, + #if linkage doesn't already exist. + $self->link_koha_record({ + record_type => $record_type, + koha_id => $linked_id, + }); + } + } + + #NOTE: Link Koha RDF to Imported RDF + if ( ($record_type && $linked_id) && ($linked_subject) ){ + my $rdf = Koha::RDF->new(); + my $context = C4::Context->new(); + my $triplestore = $context->triplestore('update') if $context && $context->triplestore('update'); + if ( $triplestore && $rdf ){ + my $koha_uri = $rdf->mint_uri($record_type, $linked_id); + my $koha_subject = RDF::Trine::Node::Resource->new($koha_uri); + $rdf->AddSeeAlso({ + model => $triplestore, + subject => $koha_subject, + object => $linked_subject, + }); + } } + #NOTE: A link will only be successful when both a MARCXML record and a RDFXML record sharing the + #same OAI-PMH repository and OAI-PMH identifier are in Koha. + #NOTE: This action is idempotent. } } --- a/Koha/OAI/Harvester/Request.pm +++ a/Koha/OAI/Harvester/Request.pm @@ -140,9 +140,28 @@ sub validate { } else { $errors->{http_url}->{malformed} = 1; } + + #NOTE: RDF + $self->_validate_import_rdf_type({ + errors => $errors, + }); + return $errors; } +#NOTE: RDF +sub _validate_import_rdf_type { + my ($self,$args) = @_; + my $errors = $args->{errors}; + my $type = $self->import_rdf_type; + if ($type){ + my $uri = URI->new($type); + if ( ! $uri->scheme ){ + $errors->{import_rdf_type}->{malformed} = 1; + } + } +} + sub _harvester { my ( $self ) = @_; my $harvester; --- a/Koha/OAI/Harvester/Worker/Download/Stream.pm +++ a/Koha/OAI/Harvester/Worker/Download/Stream.pm @@ -149,6 +149,7 @@ sub do_work { matcher_code => $import_parameters->{matcher_code}, frameworkcode => $import_parameters->{frameworkcode}, record_type => $import_parameters->{record_type}, + rdf_type => $import_parameters->{rdf_type}, }; eval { my $json_result = to_json($result, { pretty => 1 }); @@ -168,6 +169,7 @@ sub do_work { matcher_code => $import_parameters->{matcher_code}, frameworkcode => $import_parameters->{frameworkcode}, record_type => $import_parameters->{record_type}, + rdf_type => $import_parameters->{rdf_type}, }; eval { my $json_result = to_json($result, { pretty => 1 }); --- a/Koha/OAI/Harvester/Worker/Import.pm +++ a/Koha/OAI/Harvester/Worker/Import.pm @@ -63,6 +63,7 @@ sub on_start { my $matcher_code = $result->{matcher_code}; my $frameworkcode = $result->{frameworkcode}; my $record_type = $result->{record_type}; + my $rdf_type = $result->{rdf_type}; #NOTE: RDF my $matcher; if ($matcher_code){ @@ -85,6 +86,7 @@ sub on_start { framework => $frameworkcode, record_type => $record_type, matcher => $matcher, + rdf_type => $rdf_type, #NOTE: RDF }); $debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n); } --- a/Koha/Schema/Result/OaiHarvesterRequest.pm +++ a/Koha/Schema/Result/OaiHarvesterRequest.pm @@ -133,6 +133,12 @@ __PACKAGE__->table("oai_harvester_requests"); is_nullable: 0 size: 45 +=head2 import_rdf_type + + data_type: 'varchar' + is_nullable: 1 + size: 255 + =cut __PACKAGE__->add_columns( @@ -186,6 +192,8 @@ __PACKAGE__->add_columns( { data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 }, "name", { data_type => "varchar", is_nullable => 0, size => 45 }, + "import_rdf_type", + { data_type => "varchar", is_nullable => 1, size => 255 }, ); =head1 PRIMARY KEY @@ -201,8 +209,8 @@ __PACKAGE__->add_columns( __PACKAGE__->set_primary_key("id"); -# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw +# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-05-30 16:44:41 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LiH5KlQNzkZtjWujcT3DIw # You can replace this text with custom code or comments, and it will be preserved on regeneration --- a/installer/data/mysql/atomicupdate/bug_10662.sql +++ a/installer/data/mysql/atomicupdate/bug_10662.sql @@ -69,5 +69,6 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_requests` ( `import_matcher_code` varchar(10) DEFAULT NULL, `interval` int(10) unsigned NOT NULL, `name` varchar(45) NOT NULL, + `import_rdf_type` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; --- a/installer/data/mysql/kohastructure.sql +++ a/installer/data/mysql/kohastructure.sql @@ -4273,6 +4273,7 @@ CREATE TABLE IF NOT EXISTS `oai_harvester_requests` ( `import_matcher_code` varchar(10) DEFAULT NULL, `interval` int(10) unsigned NOT NULL, `name` varchar(45) NOT NULL, + `import_rdf_type` varchar(255) DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt @@ -62,6 +62,9 @@ else if (data == "updated"){ display_string = _("Updated"); } + else if (data == "unknown"){ + display_string = _("Unknown"); + } return display_string; } }, --- a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt @@ -214,6 +214,12 @@ See record matching rules to add or edit rules. +
  • + + + If you're importing RDF, you need to specify a type for the primary entity or your import will fail. + [% IF (errors.import_rdf_type.malformed) %][This must be a properly formatted URI.][% END %] +
  • --- a/t/db_dependent/OAI/Harvester/Importer/rdf.t +++ a/t/db_dependent/OAI/Harvester/Importer/rdf.t @@ -0,0 +1,53 @@ +#!/usr/bin/perl + +# Copyright 2017 Prosentient Systems +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . +# + +use Modern::Perl; +use Test::More tests => 2; + +use_ok("Koha::OAI::Harvester::Import::RDFXML"); + +subtest 'find primary entity' => sub { + + my $store = RDF::Trine::Store::Memory->new(); + my $model = RDF::Trine::Model->new($store); + + my $subject = RDF::Trine::Node::Resource->new("http://koha-community/resource/1"); + my $predicate = RDF::Trine::Node::Resource->new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); + my $object = RDF::Trine::Node::Resource->new("http://koha-community/vocab/type/record"); + my $triple = RDF::Trine::Statement->new($subject,$predicate,$object); + $model->add_statement($triple); + my $other_predicate = RDF::Trine::Node::Resource->new("http://koha-community/vocab/hasAuthor"); + my $other_object = RDF::Trine::Node::Resource->new("http://koha-community/author/david-cook"); + my $other_triple = RDF::Trine::Statement->new($subject,$other_predicate,$other_object); + $model->add_statement($other_triple); + + my $resource2_subject = RDF::Trine::Node::Resource->new("http://koha-community/resource/2"); + my $resource2_predicate = RDF::Trine::Node::Resource->new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); + my $resource2_object = RDF::Trine::Node::Resource->new("http://koha-community/vocab/type/awesome"); + my $resource2_triple = RDF::Trine::Statement->new($resource2_subject, $resource2_predicate, $resource2_object); + $model->add_statement($resource2_triple); + + my $importer = Koha::OAI::Harvester::Import::RDFXML->new(); + my $primary_subject = $importer->_get_primary_subject({ + model => $model, + type => 'http://koha-community/vocab/type/record', + }); + is($primary_subject && $primary_subject->as_string, "","Correctly found primary entity"); +}; --- a/tools/oai-pmh-harvester/dashboard.pl +++ a/tools/oai-pmh-harvester/dashboard.pl @@ -78,6 +78,7 @@ if ( ($op eq "send") && $id ){ frameworkcode => $request->import_framework_code, matcher_code => $request->import_matcher_code, record_type => $request->import_record_type, + rdf_type => $request->import_rdf_type, #NOTE: RDF }, }, }; @@ -131,4 +132,4 @@ else { $template->{VARS}->{ harvester }->{ offline } = 1; } -output_html_with_http_headers($input, $cookie, $template->output); +output_html_with_http_headers($input, $cookie, $template->output); --- a/tools/oai-pmh-harvester/request.pl +++ a/tools/oai-pmh-harvester/request.pl @@ -63,6 +63,7 @@ my $import_filter = $input->param('import_filter') // 'default'; my $import_framework_code = $input->param('import_framework_code'); my $import_record_type = $input->param('import_record_type'); my $import_matcher_code = $input->param('import_matcher_code'); +my $import_rdf_type = $input->param('import_rdf_type'); #NOTE: RDF my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0; my $name = $input->param("name"); @@ -89,6 +90,7 @@ if ($request){ import_framework_code => $import_framework_code, import_record_type => $import_record_type, import_matcher_code => $import_matcher_code, + import_rdf_type => $import_rdf_type, interval => $interval, }); } --