From ddd3608fd658fd2e239069b93b5d3c5dbf1aff5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Demians?= Date: Sun, 3 Nov 2013 17:05:38 +0100 Subject: [PATCH] Bug 11190 sitemap.pl -- Generate a Catalog sitemap Add a script sitemap.pl to process all biblio records from a Koha instance and generate Sitemap files complying with this protocol as described on http://sitemaps.org. The goal of this script is to be able to provide to search engines direct access to biblio records. It avoid leaving search engine browsing Koha OPAC and so generating a lot of traffic, and workload, for a bad result. --- C4/Installer/PerlDependencies.pm | 5 ++ Koha/Sitemaper.pm | 82 ++++++++++++++++++++++++++++ Koha/Sitemaper/Writer.pm | 112 ++++++++++++++++++++++++++++++++++++++ misc/cronjobs/sitemap.pl | 98 +++++++++++++++++++++++++++++++++ 4 files changed, 297 insertions(+), 0 deletions(-) create mode 100644 Koha/Sitemaper.pm create mode 100644 Koha/Sitemaper/Writer.pm create mode 100755 misc/cronjobs/sitemap.pl diff --git a/C4/Installer/PerlDependencies.pm b/C4/Installer/PerlDependencies.pm index b769c48..943993b 100644 --- a/C4/Installer/PerlDependencies.pm +++ b/C4/Installer/PerlDependencies.pm @@ -629,6 +629,11 @@ our $PERL_DEPS = { 'required' => '0', 'min_ver' => '2.13', }, + 'AnyEvent::HTTP' => { + 'usage' => 'Command line scripts', + 'required' => '0', + 'min_ver' => '0.003', + }, 'Moose' => { 'usage' => 'Core', 'required' => '0', diff --git a/Koha/Sitemaper.pm b/Koha/Sitemaper.pm new file mode 100644 index 0000000..58614c0 --- /dev/null +++ b/Koha/Sitemaper.pm @@ -0,0 +1,82 @@ +package Koha::Sitemaper; + +# +# Copyright 2013 Tamil s.a.r.l. +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Moose; + +extends 'AnyEvent::Processor'; + +use Modern::Perl; +use Koha::Sitemaper::Writer; +use C4::Context; + + +has url => ( is => 'rw', isa => 'Str' ); + +has verbose => ( is => 'rw', isa => 'Bool', default => 0 ); + +has sth => ( is => 'rw' ); + +has writer => ( is => 'rw', isa => 'Koha::Sitemaper::Writer' ); + + +before 'run' => sub { + my $self = shift; + + $self->writer( Koha::Sitemaper::Writer->new( url => $self->url ) ); + + my $sth = C4::Context->dbh->prepare( + "SELECT biblionumber, timestamp FROM biblio" ); + $sth->execute(); + $self->sth( $sth ); +}; + + +override 'process' => sub { + my $self = shift; + + my ($biblionumber, $timestamp) = $self->sth->fetchrow; + return unless $biblionumber; + + $self->writer->write($biblionumber, $timestamp); + return super(); +}; + + +before 'end_process' => sub { shift->writer->end(); }; + + +override 'start_message' => sub { + say "Creation of Sitemap files"; +}; + + +override 'end_message' => sub { + my $self = shift; + say "Number of biblio records processed: ", $self->count, "\n" . + "Number of Sitemap files: ", $self->writer->count; +}; + + +no Moose; +__PACKAGE__->meta->make_immutable; + + +1; + diff --git a/Koha/Sitemaper/Writer.pm b/Koha/Sitemaper/Writer.pm new file mode 100644 index 0000000..3423887 --- /dev/null +++ b/Koha/Sitemaper/Writer.pm @@ -0,0 +1,112 @@ +package Koha::Sitemaper::Writer; + +# +# Copyright 2013 Tamil s.a.r.l. +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +use Moose; +use Modern::Perl; +use XML::Writer; +use IO::File; +use DateTime; + + +my $MAX = 50000; + +has url => ( is => 'rw', isa => 'Str'); + +has current => ( is => 'rw', isa => 'Int', default => $MAX ); + +has count => ( is => 'rw', isa => 'Int', default => 0 ); + +has writer => ( is => 'rw', isa => 'XML::Writer' ); + + + +sub _writer_create { + my ($self, $name) = @_; + my $fh = IO::File->new(">$name"); + my $writer = XML::Writer->new( + OUTPUT => $fh, + DATA_MODE => 1, + DATA_INDENT => 2, + ); + return $writer; +} + + +sub _writer_end { + my $self = shift; + return unless $self->writer; + $self->writer->endTag(); + $self->writer->end(); +} + + +sub write { + my ($self, $biblionumber, $timestamp) = @_; + + if ( $self->current == $MAX ) { + $self->_writer_end(); + $self->count( $self->count + 1 ); + my $w = $self->_writer_create( sprintf("sitemap%04d.xml", $self->count) ); + $w->startTag('urlset', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'); + $self->writer($w); + $self->current(0); + } + + $self->current( $self->current + 1 ); + my $writer = $self->writer; + $writer->startTag('url'); + $writer->startTag('loc'); + $writer->characters($self->url . "/bib/$biblionumber"); + $writer->endTag(); + $writer->startTag('lastmod'); + $timestamp = substr($timestamp, 0, 10); + $writer->characters($timestamp); + $writer->endTag(); + $writer->endTag(); +} + + +sub end { + my $self = shift; + + $self->_writer_end(); + + my $w = $self->_writer_create("sitemapindex.xml"); + $w->startTag('sitemapindex', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9'); + my $now = DateTime->now()->ymd; + for my $i ( 1..$self->count ) { + $w->startTag('sitemap'); + $w->startTag('loc'); + my $name = sprintf("sitemap%04d.xml", $i); + $w->characters($self->url . "/$name"); + $w->endTag(); + $w->startTag('lastmod'); + $w->characters($now); + $w->endTag(); + $w->endTag(); + } + $w->endTag(); +} + + +no Moose; +1; + diff --git a/misc/cronjobs/sitemap.pl b/misc/cronjobs/sitemap.pl new file mode 100755 index 0000000..cc35964 --- /dev/null +++ b/misc/cronjobs/sitemap.pl @@ -0,0 +1,98 @@ +#!/usr/bin/perl + +# +# Copyright 2013 Tamil s.a.r.l. +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 2 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +package Main; + +use Modern::Perl; +use utf8; +use Pod::Usage; +use Getopt::Long; +use Koha::Sitemaper; + + +my ($verbose, $help, $url) = (0, 0, ''); +GetOptions( + 'verbose' => \$verbose, + 'help' => \$help, + 'url=s' => \$url, +); + +sub usage { + pod2usage( -verbose => 2 ); + exit; +} + + +usage() if $help || !$url; + +my $sitemaper = Koha::Sitemaper->new( + verbose => $verbose, + url => $url, +); +$sitemaper->run(); + + +=head1 USAGE + +=over + +=item sitemap.pl [--verbose|--help] --url=F + +=back + +=head1 DESCRIPTION + +Process all biblio records from a Koha instance and generate Sitemap files +complying with this protocol as described on L. The goal of +this script is to be able to provide to search engines direct access to biblio +records. It avoid leaving search engine browsing Koha OPAC and so generating +a lot of traffic, and workload, for a bad result. + +A file name F is generated. It contains references to Sitemap +multiples files. Each file contains at most 50,000 urls, and is named +F. + +The files must be stored on Koha OPAC root directoty, ie +F</koha-tmpl/>. Place also in this directory a F file +like this one: + + Sitemap: sitemapindex.xml + User-agent: * + Disallow: /cgi-bin/ + +=head1 PARAMETERS + +=over + +=item B<--url=Koha OPAC base URL> + +=item B<--verbose|-v> + +Enable script verbose mode. + +=item B<--help|-h> + +Print this help page. + +=back + +=cut + -- 1.7.2.5