Attachment #78572 for bug #10662

View | Details | Raw Unified | Return to bug 10662
Collapse All | Expand All

Lines 19-24 package Koha::OAI::Harvester::Import::RDFXML; Link Here

(-)a/Koha/OAI/Harvester/Import/RDFXML.pm (-6 / +118 lines)
19	#	19	#
20		20
21	use Modern::Perl;	21	use Modern::Perl;
		22	use Encode;
		23	use RDF::Trine;
		24	use RDF::Query;
		25
		26	use C4::Context;
		27	use Koha::RDF;
22		28
23	sub new {	29	sub new {
24	my ($class, $args) = @_;	30	my ($class, $args) = @_;
Lines 26-46 sub new { Link Here
26	return bless ($args, $class);	32	return bless ($args, $class);
27	}	33	}
28		34
		35	sub _parse_rdf_into_model {
		36	my ($self, $args) = @_;
		37
		38	#Create a RDFXML parser
		39	my $parser = RDF::Trine::Parser->new('rdfxml');
		40
		41	#Create a model in memory into which we can parse the incoming triples for processing
		42	my $store = RDF::Trine::Store::Memory->new();
		43	my $model = RDF::Trine::Model->new($store);
		44
		45	my $dom = $self->{dom};
		46
		47	my $byte_string = $dom->toString(2);
		48
		49	#Change a byte string to a character string
		50	my $character_string = Encode::decode('UTF-8', $byte_string, Encode::FB_CROAK);
		51	#NOTE: RDF::Trine::Parser::RDFXML expects characters and converts them to bytes, so
		52	#we have to change to characters before parsing.
		53
		54	#FIXME: Should you provide a base uri? Unfortunately, there's no way to do this reliably over OAI-PMH.
		55	my $base_uri = undef;
		56
		57	#NOTE: UTF-8 encoding problems have been found when using XML::SAX::PurePerl. Make sure
		58	#that you're using a parser like XML::LibXML::SAX::Parser instead. You can verify using
		59	#sax_parser_print.pl.
		60	$parser->parse_into_model($base_uri, $character_string, $model);
		61
		62	return $model;
		63	}
		64
		65	sub _get_primary_subject {
		66	my ($self,$args) = @_;
		67
		68	my $primary_subject;
		69	my $model = $args->{model};
		70	my $type = $args->{type};
		71
		72	my $object = RDF::Trine::Node::Resource->new($type);
		73	my $object_string = $object ? $object->as_string : '?object';
		74
		75	#NOTE: Select only a subject with a particular rdf:type, and filter out
		76	#any matches which are also objects in the same model. This is essential
		77	#for weeding out triples that share the same rdf:type value but are linked
		78	#objects rather than the primary subject.
		79	my $sparql = qq~
		80	SELECT DISTINCT ?subject
		81	WHERE {
		82	?subject <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> $object_string .
		83	FILTER NOT EXISTS { ?os ?op ?subject }
		84	}
		85	~;
		86	my $query = RDF::Query->new($sparql);
		87	my $iterator = $query->execute($model);
		88	if ($iterator){
		89	my @all = $iterator->get_all;
		90	if (scalar @all == 1 ){
		91	my $primary = shift @all;
		92	$primary_subject = $primary->{subject};
		93	}
		94	}
		95	return $primary_subject;
		96	}
		97
29	sub import_record {	98	sub import_record {
30	my ($self,$args) = @_;	99	my ($self,$args) = @_;
31	my $framework = $args->{framework};
32	my $record_type = $args->{record_type};	100	my $record_type = $args->{record_type};
33	my $matcher = $args->{matcher};
34	my $koha_id = $args->{koha_id};	101	my $koha_id = $args->{koha_id};
		102	my $rdf_type = $args->{rdf_type};
		103	my $oai_pmh_repository = $args->{oai_pmh_repository};
		104	my $oai_pmh_identifier = $args->{oai_pmh_identifier};
35		105
36	my $action = "error";	106	my $action = "error";
37		107
38	if (! $koha_id && $matcher){	108	#Create a model for our Koha triplestore backend
39	#Create minimal MARCXML record from RDFXML to be used for Zebra-based matching.	109	my $context = C4::Context->new();
40	}	110	my $real_model = $context->triplestore('update') if $context && $context->triplestore('update');
		111	my $memory_model = $self->_parse_rdf_into_model();
		112	my $iterator = $memory_model->as_stream;
		113	if ($iterator){
		114	if ($memory_model && $rdf_type){
		115	my $incoming_subject = $self->_get_primary_subject({
		116	model => $memory_model,
		117	type => $rdf_type,
		118	});
		119
		120	if ($incoming_subject && $incoming_subject->is_resource){
		121	my $rdf = Koha::RDF->new();
		122	if ($rdf){
		123	$real_model->begin_bulk_ops;
		124
		125	$rdf->DelNamedGraph({
		126	model => $real_model,
		127	context => $incoming_subject,
		128	});
		129	$rdf->AddNamedGraph({
		130	model => $real_model,
		131	context => $incoming_subject,
		132	iterator => $iterator,
		133	});
41		134
		135	#NOTE: Insert your own triples containing references to the OAI-PMH repository and the OAI-PMH identifier
		136	if ($oai_pmh_repository){
		137	my $repository_predicate = RDF::Trine::Node::Resource->new('http://koha-community.org/vocab/oai-pmh/repository');
		138	my $repository_object = RDF::Trine::Node::Resource->new($oai_pmh_repository);
		139	my $repository_quad = RDF::Trine::Statement::Quad->new($incoming_subject, $repository_predicate, $repository_object, $incoming_subject);
		140	$real_model->add_statement($repository_quad);
		141	}
		142	if ($oai_pmh_identifier){
		143	my $identifier_predicate = RDF::Trine::Node::Resource->new('http://koha-community.org/vocab/oai-pmh/identifier');
		144	my $identifier_object = RDF::Trine::Node::Resource->new($oai_pmh_identifier);
		145	my $identifier_quad = RDF::Trine::Statement::Quad->new($incoming_subject, $identifier_predicate, $identifier_object, $incoming_subject);
		146	$real_model->add_statement($identifier_quad);
		147	}
42		148
43	#TODO: add/update triplestore	149	$real_model->end_bulk_ops;
		150
		151	$action = 'unknown'; #FIXME: RDF::Trine doesn't return meaningful values so we can't really say much else...
		152	}
		153	}
		154	}
		155	}
44	return ($action,$koha_id);	156	return ($action,$koha_id);
45	}	157	}
46		158




use XML::LibXSLT;
use URI;
use File::Basename;
use RDF::Query;

use C4::Context;
use C4::Biblio;

use Koha::Database;
use Koha::OAI::Harvester::Import::MARCXML;
use Koha::OAI::Harvester::Import::RDFXML;
use Koha::RDF;

my $schema = Koha::Database->new()->schema();


    return $link_id;
}


sub _find_rdf_link {
    my ($self, $args) = @_;
    my ($subject,$graph);
    my $repository = $self->{repository};
    my $identifier = $self->{header_identifier};

    my $context = C4::Context->new;
    my $triplestore = $context->triplestore('query');
    if ($triplestore){
        if ( URI->new($repository) && URI->new($identifier) ){
            my $sparql = qq~
                PREFIX koha-oai: <http://koha-community.org/vocab/oai-pmh/>
                SELECT ?g ?s ?identifier ?repository
                WHERE {
                    GRAPH ?g {
                        ?s koha-oai:identifier ?identifier .
                        ?s koha-oai:repository ?repository .
                        ?s koha-oai:identifier <$identifier> .
                        ?s koha-oai:repository <$repository> .
                    }
                }
            ~;
            my $query = RDF::Query->new($sparql);
            my $iterator = $query->execute($triplestore);
            my @rows = $iterator->get_all;
            if (scalar @rows == 1){
                my $row = $rows[0];
                $subject = $row->{s};
                $graph = $row->{g};
            }
        }
    }
    return ($subject,$graph);
}

=head3

    my ($action,$record_id) = $oai_record->import_record({

    my $framework = $args->{framework} || '';
    my $record_type = $args->{record_type} || 'biblio';
    my $matcher = $args->{matcher};
    my $rdf_type = $args->{rdf_type}; #NOTE: RDF

    my $action = "error";

    #NOTE: RDF
    #Find linkage between OAI-PMH repository-identifier and Koha record id
    my $linked_id = $self->_find_koha_link({
        record_type => $record_type,
    });

    #NOTE: RDF
    my $context = C4::Context->new();
    #Find linkage between OAI-PMH repository-identifier and RDF records in the triplestore
    my ($linked_subject, $linked_graph) = $self->_find_rdf_link if $context && $context->triplestore('query');

    if ($self->is_deleted_upstream){
        #FIXME: If a record is deleted upstream, it will not contain a metadata element, so we don't know what metadata
        #format we requested. Perhaps we should be recording the metadataPrefix during download and import, so we
        #can have more targeted deletes.
        #NOTE: "Other records, with different metadataPrefix but the same unique identifier, may remain available for the item."
        #https://www.openarchives.org/OAI/openarchivesprotocol.html#DeletedRecords

        if ($linked_graph){
            my $model = $context->triplestore('update') if $context && $context->triplestore('update');
            my $rdf = Koha::RDF->new();
            if ($rdf){
                $model->begin_bulk_ops;
                $rdf->DelNamedGraph({
                    model => $model,
                    context => $linked_graph,
                });
                $model->end_bulk_ops;
            }
            #NOTE: We don't set action here, since RDF::Trine doesn't return a useful value, plus
            #the RDF is meaningless without the Koha record mentioned below anyway.
        }

        if ($linked_id){
            $action = $self->delete_koha_record({
                record_id => $linked_id,
                record_type => $record_type,
            });
            #TODO: How to handle RDF deletions?
            #NOTE: If we delete the Koha record, then we lose the link to our downloaded record, so I don't know if it matters...
            #...although surely we'd want some way of cleaning up records that aren't needed anymore? I suppose that could come up with a cronjob that scans for triples that aren't linked to...
        }
        else {
            $action = "not_found";
            #NOTE: If there's no OAI-PMH repository-identifier pair in the database,
            #then there's no perfect way to find a linked record to delete.
        }
        #NOTE: RDF
        #TODO: Delete named graph found via $linked_graph...
        #TODO: Do we also unlink the Koha record from this OAI record? I suppose so...
    }
    else {
        $self->set_filter($filter);

        my $import_record = $self->filter();

        if ($import_record){
        #NOTE: You can inspect the object's class here for special type handling if necessary...
            ($action,$linked_id) = $import_record->import_record({
                oai_pmh_repository => $self->{repository},
                oai_pmh_identifier => $self->{header_identifier},
                framework => $framework,
                record_type => $record_type,
                matcher => $matcher,
                koha_id => $linked_id,
                linked_subject => $linked_subject, #NOTE: RDF
                rdf_type => $rdf_type, #NOTE: RDF
            });

            if ($import_record->isa('Koha::Import::MARCXML')){
                if ($linked_id){
                    #Link Koha record ID to OAI-PMH details for this record type,
                    #if linkage doesn't already exist.
                    $self->link_koha_record({
                        record_type => $record_type,
                        koha_id => $linked_id,
                    });
                }
            }

            #NOTE: Link Koha RDF to Imported RDF
            if ( ($record_type && $linked_id) && ($linked_subject) ){
                my $rdf = Koha::RDF->new();
                my $context = C4::Context->new();
                my $triplestore = $context->triplestore('update') if $context && $context->triplestore('update');
                if ( $triplestore && $rdf ){
                    my $koha_uri = $rdf->mint_uri($record_type, $linked_id);
                    my $koha_subject = RDF::Trine::Node::Resource->new($koha_uri);
                    $rdf->AddSeeAlso({
                        model => $triplestore,
                        subject => $koha_subject,
                        object => $linked_subject,
                    });
                }
            }
            #NOTE: A link will only be successful when both a MARCXML record and a RDFXML record sharing the
            #same OAI-PMH repository and OAI-PMH identifier are in Koha.
            #NOTE: This action is idempotent.
        }
    }





    } else {
        $errors->{http_url}->{malformed} = 1;
    }

    #NOTE: RDF
    $self->_validate_import_rdf_type({
        errors => $errors,
    });

    return $errors;
}

#NOTE: RDF
sub _validate_import_rdf_type {
    my ($self,$args) = @_;
    my $errors = $args->{errors};
    my $type = $self->import_rdf_type;
    if ($type){
        my $uri = URI->new($type);
        if ( ! $uri->scheme ){
            $errors->{import_rdf_type}->{malformed} = 1;
        }
    }
}

sub _harvester {
    my ( $self ) = @_;
    my $harvester;

Lines 149-154 sub do_work { Link Here

(-)a/Koha/OAI/Harvester/Worker/Download/Stream.pm (+2 lines)
149	matcher_code => $import_parameters->{matcher_code},	149	matcher_code => $import_parameters->{matcher_code},
150	frameworkcode => $import_parameters->{frameworkcode},	150	frameworkcode => $import_parameters->{frameworkcode},
151	record_type => $import_parameters->{record_type},	151	record_type => $import_parameters->{record_type},
		152	rdf_type => $import_parameters->{rdf_type},
152	};	153	};
153	eval {	154	eval {
154	my $json_result = to_json($result, { pretty => 1 });	155	my $json_result = to_json($result, { pretty => 1 });
Lines 168-173 sub do_work { Link Here
168	matcher_code => $import_parameters->{matcher_code},	169	matcher_code => $import_parameters->{matcher_code},
169	frameworkcode => $import_parameters->{frameworkcode},	170	frameworkcode => $import_parameters->{frameworkcode},
170	record_type => $import_parameters->{record_type},	171	record_type => $import_parameters->{record_type},
		172	rdf_type => $import_parameters->{rdf_type},
171	};	173	};
172	eval {	174	eval {
173	my $json_result = to_json($result, { pretty => 1 });	175	my $json_result = to_json($result, { pretty => 1 });

Lines 63-68 sub on_start { Link Here

(-)a/Koha/OAI/Harvester/Worker/Import.pm (+2 lines)
63	my $matcher_code = $result->{matcher_code};	63	my $matcher_code = $result->{matcher_code};
64	my $frameworkcode = $result->{frameworkcode};	64	my $frameworkcode = $result->{frameworkcode};
65	my $record_type = $result->{record_type};	65	my $record_type = $result->{record_type};
		66	my $rdf_type = $result->{rdf_type}; #NOTE: RDF
66		67
67	my $matcher;	68	my $matcher;
68	if ($matcher_code){	69	if ($matcher_code){
Lines 85-90 sub on_start { Link Here
85	framework => $frameworkcode,	86	framework => $frameworkcode,
86	record_type => $record_type,	87	record_type => $record_type,
87	matcher => $matcher,	88	matcher => $matcher,
		89	rdf_type => $rdf_type, #NOTE: RDF
88	});	90	});
89	$debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);	91	$debug && print STDOUT qq({ "import_result": { "task_uuid": "$task_uuid", "action": "$action", "filename": "$filename", "koha_id": "$linked_id" } }\n);
90	}	92	}

Lines 133-138 __PACKAGE__->table("oai_harvester_requests"); Link Here

(-)a/Koha/Schema/Result/OaiHarvesterRequest.pm (-2 / +10 lines)
133	is_nullable: 0	133	is_nullable: 0
134	size: 45	134	size: 45
135		135
		136	=head2 import_rdf_type
		137
		138	data_type: 'varchar'
		139	is_nullable: 1
		140	size: 255
		141
136	=cut	142	=cut
137		143
138	__PACKAGE__->add_columns(	144	__PACKAGE__->add_columns(
Lines 186-191 __PACKAGE__->add_columns( Link Here
186	{ data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },	192	{ data_type => "integer", extra => { unsigned => 1 }, is_nullable => 0 },
187	"name",	193	"name",
188	{ data_type => "varchar", is_nullable => 0, size => 45 },	194	{ data_type => "varchar", is_nullable => 0, size => 45 },
		195	"import_rdf_type",
		196	{ data_type => "varchar", is_nullable => 1, size => 255 },
189	);	197	);
190		198
191	=head1 PRIMARY KEY	199	=head1 PRIMARY KEY
Lines 201-208 __PACKAGE__->add_columns( Link Here
201	__PACKAGE__->set_primary_key("id");	209	__PACKAGE__->set_primary_key("id");
202		210
203		211
204	# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-04-07 11:26:24	212	# Created by DBIx::Class::Schema::Loader v0.07046 @ 2017-05-30 16:44:41
205	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Vm/b4yurmr8WF7z+Fo6KEw	213	# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:LiH5KlQNzkZtjWujcT3DIw
206		214
207		215
208	# You can replace this text with custom code or comments, and it will be preserved on regeneration	216	# You can replace this text with custom code or comments, and it will be preserved on regeneration

Lines 69-73 CREATE TABLE IF NOT EXISTS `oai_harvester_requests` ( Link Here

(-)a/installer/data/mysql/atomicupdate/bug_10662.sql (+1 lines)
69	`import_matcher_code` varchar(10) DEFAULT NULL,	69	`import_matcher_code` varchar(10) DEFAULT NULL,
70	`interval` int(10) unsigned NOT NULL,	70	`interval` int(10) unsigned NOT NULL,
71	`name` varchar(45) NOT NULL,	71	`name` varchar(45) NOT NULL,
		72	`import_rdf_type` varchar(255) DEFAULT NULL,
72	PRIMARY KEY (`id`)	73	PRIMARY KEY (`id`)
73	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;	74	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;

Lines 4263-4268 CREATE TABLE IF NOT EXISTS `oai_harvester_requests` ( Link Here

(-)a/installer/data/mysql/kohastructure.sql (+1 lines)
4263	`import_matcher_code` varchar(10) DEFAULT NULL,	4263	`import_matcher_code` varchar(10) DEFAULT NULL,
4264	`interval` int(10) unsigned NOT NULL,	4264	`interval` int(10) unsigned NOT NULL,
4265	`name` varchar(45) NOT NULL,	4265	`name` varchar(45) NOT NULL,
		4266	`import_rdf_type` varchar(255) DEFAULT NULL,
4266	PRIMARY KEY (`id`)	4267	PRIMARY KEY (`id`)
4267	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;	4268	) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
4268		4269

Lines 62-67 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/dashboard.tt (+3 lines)
62	else if (data == "updated"){	62	else if (data == "updated"){
63	display_string = _("Updated");	63	display_string = _("Updated");
64	}	64	}
		65	else if (data == "unknown"){
		66	display_string = _("Unknown");
		67	}
65	return display_string;	68	return display_string;
66	}	69	}
67	},	70	},

Lines 214-219 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/oai-pmh-harvester/request.tt (+6 lines)
214	</select>	214	</select>
215	<span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>	215	<span class="help">See <a href="/cgi-bin/koha/admin/matching-rules.pl">record matching rules</a> to add or edit rules.</span>
216	</li>	216	</li>
		217	<li>
		218	<label for="import_rdf_type">RDF type</label>
		219	<input type="text" size="30" id="import_rdf_type" name="import_rdf_type" value="[% oai_pmh_request.import_rdf_type %]">
		220	<span class="help">If you're importing RDF, you need to specify a type for the primary entity or your import will fail.</span>
		221	[% IF (errors.import_rdf_type.malformed) %]<span class="error">[This must be a properly formatted URI.]</span>[% END %]
		222	</li>
217	</ol>	223	</ol>
218	</fieldset>	224	</fieldset>
219	<fieldset class="rows">	225	<fieldset class="rows">




#!/usr/bin/perl

# Copyright 2017 Prosentient Systems
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <http://www.gnu.org/licenses>.
#

use Modern::Perl;
use Test::More tests => 2;

use_ok("Koha::OAI::Harvester::Import::RDFXML");

subtest 'find primary entity' => sub {

    my $store = RDF::Trine::Store::Memory->new();
    my $model = RDF::Trine::Model->new($store);

    my $subject = RDF::Trine::Node::Resource->new("http://koha-community/resource/1");
    my $predicate = RDF::Trine::Node::Resource->new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
    my $object = RDF::Trine::Node::Resource->new("http://koha-community/vocab/type/record");
    my $triple = RDF::Trine::Statement->new($subject,$predicate,$object);
    $model->add_statement($triple);
    my $other_predicate = RDF::Trine::Node::Resource->new("http://koha-community/vocab/hasAuthor");
    my $other_object = RDF::Trine::Node::Resource->new("http://koha-community/author/david-cook");
    my $other_triple = RDF::Trine::Statement->new($subject,$other_predicate,$other_object);
    $model->add_statement($other_triple);

    my $resource2_subject = RDF::Trine::Node::Resource->new("http://koha-community/resource/2");
    my $resource2_predicate = RDF::Trine::Node::Resource->new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
    my $resource2_object = RDF::Trine::Node::Resource->new("http://koha-community/vocab/type/awesome");
    my $resource2_triple = RDF::Trine::Statement->new($resource2_subject, $resource2_predicate, $resource2_object);
    $model->add_statement($resource2_triple);

    my $importer = Koha::OAI::Harvester::Import::RDFXML->new();
    my $primary_subject = $importer->_get_primary_subject({
        model => $model,
        type => 'http://koha-community/vocab/type/record',
    });
    is($primary_subject && $primary_subject->as_string, "<http://koha-community/resource/1>","Correctly found primary entity");
};

Lines 78-83 if ( ($op eq "send") && $id ){ Link Here

(-)a/tools/oai-pmh-harvester/dashboard.pl (-1 / +2 lines)
78	frameworkcode => $request->import_framework_code,	78	frameworkcode => $request->import_framework_code,
79	matcher_code => $request->import_matcher_code,	79	matcher_code => $request->import_matcher_code,
80	record_type => $request->import_record_type,	80	record_type => $request->import_record_type,
		81	rdf_type => $request->import_rdf_type, #NOTE: RDF
81	},	82	},
82	},	83	},
83	};	84	};
Lines 131-134 else { Link Here
131	$template->{VARS}->{ harvester }->{ offline } = 1;	132	$template->{VARS}->{ harvester }->{ offline } = 1;
132	}	133	}
133		134
134	output_html_with_http_headers($input, $cookie, $template->output);	135	output_html_with_http_headers($input, $cookie, $template->output);

Lines 63-68 my $import_filter = $input->param('import_filter') // 'default'; Link Here

(-)a/tools/oai-pmh-harvester/request.pl (-1 / +2 lines)
63	my $import_framework_code = $input->param('import_framework_code');	63	my $import_framework_code = $input->param('import_framework_code');
64	my $import_record_type = $input->param('import_record_type');	64	my $import_record_type = $input->param('import_record_type');
65	my $import_matcher_code = $input->param('import_matcher_code');	65	my $import_matcher_code = $input->param('import_matcher_code');
		66	my $import_rdf_type = $input->param('import_rdf_type'); #NOTE: RDF
66		67
67	my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;	68	my $interval = $input->param("interval") ? int ( $input->param("interval") ) : 0;
68	my $name = $input->param("name");	69	my $name = $input->param("name");
Lines 89-94 if ($request){ Link Here
89	import_framework_code => $import_framework_code,	90	import_framework_code => $import_framework_code,
90	import_record_type => $import_record_type,	91	import_record_type => $import_record_type,
91	import_matcher_code => $import_matcher_code,	92	import_matcher_code => $import_matcher_code,
		93	import_rdf_type => $import_rdf_type,
92	interval => $interval,	94	interval => $interval,
93	});	95	});
94	}	96	}
95	-

Return to bug 10662

Lines 23-28 use XML::LibXML; Link Here

(-)a/Koha/OAI/Harvester/Import/Record.pm (-11 / +102 lines)
23	use XML::LibXSLT;	23	use XML::LibXSLT;
24	use URI;	24	use URI;
25	use File::Basename;	25	use File::Basename;
		26	use RDF::Query;
26		27
27	use C4::Context;	28	use C4::Context;
28	use C4::Biblio;	29	use C4::Biblio;
Lines 30-35 use C4::Biblio; Link Here
30	use Koha::Database;	31	use Koha::Database;
31	use Koha::OAI::Harvester::Import::MARCXML;	32	use Koha::OAI::Harvester::Import::MARCXML;
32	use Koha::OAI::Harvester::Import::RDFXML;	33	use Koha::OAI::Harvester::Import::RDFXML;
		34	use Koha::RDF;
33		35
34	my $schema = Koha::Database->new()->schema();	36	my $schema = Koha::Database->new()->schema();
35		37
Lines 179-184 sub _find_koha_link { Link Here
179	return $link_id;	181	return $link_id;
180	}	182	}
181		183
		184
		185	sub _find_rdf_link {
		186	my ($self, $args) = @_;
		187	my ($subject,$graph);
		188	my $repository = $self->{repository};
		189	my $identifier = $self->{header_identifier};
		190
		191	my $context = C4::Context->new;
		192	my $triplestore = $context->triplestore('query');
		193	if ($triplestore){
		194	if ( URI->new($repository) && URI->new($identifier) ){
		195	my $sparql = qq~
		196	PREFIX koha-oai: <http://koha-community.org/vocab/oai-pmh/>
		197	SELECT ?g ?s ?identifier ?repository
		198	WHERE {
		199	GRAPH ?g {
		200	?s koha-oai:identifier ?identifier .
		201	?s koha-oai:repository ?repository .
		202	?s koha-oai:identifier <$identifier> .
		203	?s koha-oai:repository <$repository> .
		204	}
		205	}
		206	~;
		207	my $query = RDF::Query->new($sparql);
		208	my $iterator = $query->execute($triplestore);
		209	my @rows = $iterator->get_all;
		210	if (scalar @rows == 1){
		211	my $row = $rows[0];
		212	$subject = $row->{s};
		213	$graph = $row->{g};
		214	}
		215	}
		216	}
		217	return ($subject,$graph);
		218	}
		219
182	=head3	220	=head3
183		221
184	my ($action,$record_id) = $oai_record->import_record({	222	my ($action,$record_id) = $oai_record->import_record({
Lines 198-227 sub import_record { Link Here
198	my $framework = $args->{framework} \|\| '';	236	my $framework = $args->{framework} \|\| '';
199	my $record_type = $args->{record_type} \|\| 'biblio';	237	my $record_type = $args->{record_type} \|\| 'biblio';
200	my $matcher = $args->{matcher};	238	my $matcher = $args->{matcher};
		239	my $rdf_type = $args->{rdf_type}; #NOTE: RDF
201		240
202	my $action = "error";	241	my $action = "error";
203		242
		243	#NOTE: RDF
204	#Find linkage between OAI-PMH repository-identifier and Koha record id	244	#Find linkage between OAI-PMH repository-identifier and Koha record id
205	my $linked_id = $self->_find_koha_link({	245	my $linked_id = $self->_find_koha_link({
206	record_type => $record_type,	246	record_type => $record_type,
207	});	247	});
208		248
		249	#NOTE: RDF
		250	my $context = C4::Context->new();
		251	#Find linkage between OAI-PMH repository-identifier and RDF records in the triplestore
		252	my ($linked_subject, $linked_graph) = $self->_find_rdf_link if $context && $context->triplestore('query');
		253
209	if ($self->is_deleted_upstream){	254	if ($self->is_deleted_upstream){
210	#NOTE: If a record is deleted upstream, it will not contain a metadata element	255	#FIXME: If a record is deleted upstream, it will not contain a metadata element, so we don't know what metadata
		256	#format we requested. Perhaps we should be recording the metadataPrefix during download and import, so we
		257	#can have more targeted deletes.
		258	#NOTE: "Other records, with different metadataPrefix but the same unique identifier, may remain available for the item."
		259	#https://www.openarchives.org/OAI/openarchivesprotocol.html#DeletedRecords
		260
		261	if ($linked_graph){
		262	my $model = $context->triplestore('update') if $context && $context->triplestore('update');
		263	my $rdf = Koha::RDF->new();
		264	if ($rdf){
		265	$model->begin_bulk_ops;
		266	$rdf->DelNamedGraph({
		267	model => $model,
		268	context => $linked_graph,
		269	});
		270	$model->end_bulk_ops;
		271	}
		272	#NOTE: We don't set action here, since RDF::Trine doesn't return a useful value, plus
		273	#the RDF is meaningless without the Koha record mentioned below anyway.
		274	}
		275
211	if ($linked_id){	276	if ($linked_id){
212	$action = $self->delete_koha_record({	277	$action = $self->delete_koha_record({
213	record_id => $linked_id,	278	record_id => $linked_id,
214	record_type => $record_type,	279	record_type => $record_type,
215	});	280	});
216	#TODO: How to handle RDF deletions?
217	#NOTE: If we delete the Koha record, then we lose the link to our downloaded record, so I don't know if it matters...
218	#...although surely we'd want some way of cleaning up records that aren't needed anymore? I suppose that could come up with a cronjob that scans for triples that aren't linked to...
219	}	281	}
220	else {	282	else {
221	$action = "not_found";	283	$action = "not_found";
222	#NOTE: If there's no OAI-PMH repository-identifier pair in the database,	284	#NOTE: If there's no OAI-PMH repository-identifier pair in the database,
223	#then there's no perfect way to find a linked record to delete.	285	#then there's no perfect way to find a linked record to delete.
224	}	286	}
		287	#NOTE: RDF
		288	#TODO: Delete named graph found via $linked_graph...
		289	#TODO: Do we also unlink the Koha record from this OAI record? I suppose so...
225	}	290	}
226	else {	291	else {
227	$self->set_filter($filter);	292	$self->set_filter($filter);
Lines 230-250 sub import_record { Link Here
230	my $import_record = $self->filter();	295	my $import_record = $self->filter();
231		296
232	if ($import_record){	297	if ($import_record){
		298	#NOTE: You can inspect the object's class here for special type handling if necessary...
233	($action,$linked_id) = $import_record->import_record({	299	($action,$linked_id) = $import_record->import_record({
		300	oai_pmh_repository => $self->{repository},
		301	oai_pmh_identifier => $self->{header_identifier},
234	framework => $framework,	302	framework => $framework,
235	record_type => $record_type,	303	record_type => $record_type,
236	matcher => $matcher,	304	matcher => $matcher,
237	koha_id => $linked_id,	305	koha_id => $linked_id,
		306	linked_subject => $linked_subject, #NOTE: RDF
		307	rdf_type => $rdf_type, #NOTE: RDF
238	});	308	});
239		309
240	if ($linked_id){	310	if ($import_record->isa('Koha::Import::MARCXML')){
241	#Link Koha record ID to OAI-PMH details for this record type,	311	if ($linked_id){
242	#if linkage doesn't already exist.	312	#Link Koha record ID to OAI-PMH details for this record type,
243	$self->link_koha_record({	313	#if linkage doesn't already exist.
244	record_type => $record_type,	314	$self->link_koha_record({
245	koha_id => $linked_id,	315	record_type => $record_type,
246	});	316	koha_id => $linked_id,
		317	});
		318	}
		319	}
		320
		321	#NOTE: Link Koha RDF to Imported RDF
		322	if ( ($record_type && $linked_id) && ($linked_subject) ){
		323	my $rdf = Koha::RDF->new();
		324	my $context = C4::Context->new();
		325	my $triplestore = $context->triplestore('update') if $context && $context->triplestore('update');
		326	if ( $triplestore && $rdf ){
		327	my $koha_uri = $rdf->mint_uri($record_type, $linked_id);
		328	my $koha_subject = RDF::Trine::Node::Resource->new($koha_uri);
		329	$rdf->AddSeeAlso({
		330	model => $triplestore,
		331	subject => $koha_subject,
		332	object => $linked_subject,
		333	});
		334	}
247	}	335	}
		336	#NOTE: A link will only be successful when both a MARCXML record and a RDFXML record sharing the
		337	#same OAI-PMH repository and OAI-PMH identifier are in Koha.
		338	#NOTE: This action is idempotent.
248	}	339	}
249	}	340	}
250		341

Lines 140-148 sub validate { Link Here

(-)a/Koha/OAI/Harvester/Request.pm (+19 lines)
140	} else {	140	} else {
141	$errors->{http_url}->{malformed} = 1;	141	$errors->{http_url}->{malformed} = 1;
142	}	142	}
		143
		144	#NOTE: RDF
		145	$self->_validate_import_rdf_type({
		146	errors => $errors,
		147	});
		148
143	return $errors;	149	return $errors;
144	}	150	}
145		151
		152	#NOTE: RDF
		153	sub _validate_import_rdf_type {
		154	my ($self,$args) = @_;
		155	my $errors = $args->{errors};
		156	my $type = $self->import_rdf_type;
		157	if ($type){
		158	my $uri = URI->new($type);
		159	if ( ! $uri->scheme ){
		160	$errors->{import_rdf_type}->{malformed} = 1;
		161	}
		162	}
		163	}
		164
146	sub _harvester {	165	sub _harvester {
147	my ( $self ) = @_;	166	my ( $self ) = @_;
148	my $harvester;	167	my $harvester;

Line 0 Link Here

(-)a/t/db_dependent/OAI/Harvester/Importer/rdf.t (+53 lines)
	1	#!/usr/bin/perl
	2
	3	# Copyright 2017 Prosentient Systems
	4	#
	5	# This file is part of Koha.
	6	#
	7	# Koha is free software; you can redistribute it and/or modify it
	8	# under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 3 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# Koha is distributed in the hope that it will be useful, but
	13	# WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with Koha; if not, see <http://www.gnu.org/licenses>.
	19	#
	20
	21	use Modern::Perl;
	22	use Test::More tests => 2;
	23
	24	use_ok("Koha::OAI::Harvester::Import::RDFXML");
	25
	26	subtest 'find primary entity' => sub {
	27
	28	my $store = RDF::Trine::Store::Memory->new();
	29	my $model = RDF::Trine::Model->new($store);
	30
	31	my $subject = RDF::Trine::Node::Resource->new("http://koha-community/resource/1");
	32	my $predicate = RDF::Trine::Node::Resource->new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
	33	my $object = RDF::Trine::Node::Resource->new("http://koha-community/vocab/type/record");
	34	my $triple = RDF::Trine::Statement->new($subject,$predicate,$object);
	35	$model->add_statement($triple);
	36	my $other_predicate = RDF::Trine::Node::Resource->new("http://koha-community/vocab/hasAuthor");
	37	my $other_object = RDF::Trine::Node::Resource->new("http://koha-community/author/david-cook");
	38	my $other_triple = RDF::Trine::Statement->new($subject,$other_predicate,$other_object);
	39	$model->add_statement($other_triple);
	40
	41	my $resource2_subject = RDF::Trine::Node::Resource->new("http://koha-community/resource/2");
	42	my $resource2_predicate = RDF::Trine::Node::Resource->new("http://www.w3.org/1999/02/22-rdf-syntax-ns#type");
	43	my $resource2_object = RDF::Trine::Node::Resource->new("http://koha-community/vocab/type/awesome");
	44	my $resource2_triple = RDF::Trine::Statement->new($resource2_subject, $resource2_predicate, $resource2_object);
	45	$model->add_statement($resource2_triple);
	46
	47	my $importer = Koha::OAI::Harvester::Import::RDFXML->new();
	48	my $primary_subject = $importer->_get_primary_subject({
	49	model => $model,
	50	type => 'http://koha-community/vocab/type/record',
	51	});
	52	is($primary_subject && $primary_subject->as_string, "<http://koha-community/resource/1>","Correctly found primary entity");
	53	};