View | Details | Raw Unified | Return to bug 11190
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
Lines 629-634 our $PERL_DEPS = { Link Here
629
        'required' => '0',
629
        'required' => '0',
630
        'min_ver'  => '2.13',
630
        'min_ver'  => '2.13',
631
    },
631
    },
632
    'AnyEvent::HTTP' => {
633
        'usage'    => 'Command line scripts',
634
        'required' => '0',
635
        'min_ver'  => '0.003',
636
    },
632
    'Moose' => {
637
    'Moose' => {
633
        'usage'    => 'Core',
638
        'usage'    => 'Core',
634
        'required' => '0',
639
        'required' => '0',
Lines 704-709 our $PERL_DEPS = { Link Here
704
        'required' => '0',
709
        'required' => '0',
705
        'min_ver'  => '0.73',
710
        'min_ver'  => '0.73',
706
    },
711
    },
712
    'XML::Writer' => {
713
        'usage'    => 'Command line scripts',
714
        'required' => '0',
715
        'min_ver'  => '0.614',
716
    },
707
};
717
};
708
718
709
1;
719
1;
(-)a/Koha/Sitemapper.pm (+96 lines)
Line 0 Link Here
1
package Koha::Sitemapper;
2
3
#
4
# Copyright 2013 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
use Moose;
22
23
extends 'AnyEvent::Processor';
24
25
use Modern::Perl;
26
use Koha::Sitemapper::Writer;
27
use C4::Context;
28
29
30
has url => ( is => 'rw', isa => 'Str' );
31
32
has dir => (
33
    is => 'rw',
34
    isa => 'Str',
35
    default => '.',
36
    trigger => sub {
37
        my ($self, $dir) = @_;
38
        unless (-d $dir) {
39
            say "This is not a valid directory: $dir";
40
            exit;
41
        }
42
    }
43
);
44
45
has short => ( is => 'rw', isa => 'Bool', default => 1 );
46
47
has verbose => ( is => 'rw', isa => 'Bool', default => 0 );
48
49
has sth => ( is => 'rw' );
50
51
has writer => ( is => 'rw', isa => 'Koha::Sitemapper::Writer' );
52
53
54
before 'run' => sub {
55
    my $self = shift;
56
57
    $self->writer( Koha::Sitemapper::Writer->new( sitemapper => $self ) );
58
    my $sth = C4::Context->dbh->prepare(
59
         "SELECT biblionumber, timestamp FROM biblio" );
60
    $sth->execute();
61
    $self->sth( $sth );
62
};
63
64
65
override 'process' => sub {
66
    my $self = shift;
67
68
    my ($biblionumber, $timestamp) = $self->sth->fetchrow;
69
    return unless $biblionumber;
70
71
    $self->writer->write($biblionumber, $timestamp);
72
    return super();
73
};
74
75
76
before 'end_process' => sub { shift->writer->end(); };
77
78
79
override 'start_message' => sub {
80
    say "Creation of Sitemap files in '" . shift->dir . "' directory";
81
};
82
83
84
override 'end_message' => sub {
85
    my $self = shift;
86
    say "Number of biblio records processed: ", $self->count, "\n" .
87
        "Number of Sitemap files:            ", $self->writer->count;
88
};
89
90
91
no Moose;
92
__PACKAGE__->meta->make_immutable;
93
94
95
1;
96
(-)a/Koha/Sitemapper/Writer.pm (+126 lines)
Line 0 Link Here
1
package Koha::Sitemapper::Writer;
2
3
#
4
# Copyright 2014 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
use Moose;
23
use Modern::Perl;
24
use XML::Writer;
25
use IO::File;
26
use DateTime;
27
28
29
my $MAX = 50000;
30
31
32
has sitemapper => (is => 'rw', isa => 'Koha::Sitemapper');
33
34
has current => ( is => 'rw', isa => 'Int', default => $MAX );
35
36
has count => ( is => 'rw', isa => 'Int', default => 0 );
37
38
has writer => ( is => 'rw', isa => 'XML::Writer' );
39
40
41
42
sub _writer_create {
43
    my ($self, $name) = @_;
44
    $name = $self->sitemapper->dir . "/$name";
45
    my $fh = IO::File->new(">/$name");
46
    unless ($fh) {
47
        say "Impossible to create file: $name";
48
        exit;
49
    }
50
    my $writer = XML::Writer->new(
51
        OUTPUT => $fh,
52
        DATA_MODE => 1,
53
        DATA_INDENT => 2,
54
    );
55
    $writer->xmlDecl("UTF-8");
56
    return $writer;
57
}
58
59
60
sub _writer_end {
61
    my $self = shift;
62
    return unless $self->writer;
63
    $self->writer->endTag();
64
    $self->writer->end();
65
}
66
67
68
sub write {
69
    my ($self, $biblionumber, $timestamp) = @_;
70
71
    if ( $self->current == $MAX ) {
72
        $self->_writer_end();
73
        $self->count( $self->count + 1 );
74
        my $w = $self->_writer_create( sprintf("sitemap%04d.xml", $self->count) );
75
        $w->startTag(
76
            'urlset',
77
            'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
78
            'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
79
            'xsi:schemaLocation' => 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd');
80
        $self->writer($w);
81
        $self->current(0);
82
    }
83
84
    $self->current( $self->current + 1 );
85
    my $writer = $self->writer;
86
    my $url = $self->sitemapper->url .
87
              ($self->sitemapper->short ? '/bib/' : '/cgi-bin/koha/opac-detail.pl?biblionumber=') .
88
              $biblionumber;
89
    $writer->startTag('url');
90
        $writer->startTag('loc');
91
            $writer->characters($url);
92
        $writer->endTag();
93
        $writer->startTag('lastmod');
94
            $timestamp = substr($timestamp, 0, 10);
95
            $writer->characters($timestamp);
96
        $writer->endTag();
97
    $writer->endTag();
98
}
99
100
101
sub end {
102
    my $self = shift;
103
104
    $self->_writer_end();
105
106
    my $w = $self->_writer_create("sitemapindex.xml");
107
    $w->startTag('sitemapindex', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9');
108
    my $now = DateTime->now()->ymd;
109
    for my $i ( 1..$self->count ) {
110
        $w->startTag('sitemap');
111
            $w->startTag('loc');
112
                my $name = sprintf("sitemap%04d.xml", $i);
113
                $w->characters($self->sitemapper->url . "/$name");
114
            $w->endTag();
115
            $w->startTag('lastmod');
116
                $w->characters($now);
117
            $w->endTag();
118
        $w->endTag();
119
    }
120
    $w->endTag();
121
}
122
123
124
no Moose;
125
1;
126
(-)a/misc/cronjobs/sitemap.pl (+134 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
#
4
# Copyright 2014 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
package Main;
23
24
use Modern::Perl;
25
use utf8;
26
use Pod::Usage;
27
use Getopt::Long;
28
use C4::Biblio;
29
use Koha::Sitemapper;
30
31
32
my ($verbose, $help, $url, $dir, $short) = (0, 0, '', '.', 1);
33
GetOptions( 
34
    'verbose'   => \$verbose,
35
    'help'      => \$help,
36
    'url=s'     => \$url,
37
    'dir=s'     => \$dir,
38
    'short!'    => \$short,
39
);
40
41
sub usage {
42
    pod2usage( -verbose => 2 );
43
    exit;
44
} 
45
46
usage() if $help;
47
48
unless ($url) {
49
    $url = C4::Context->preference("OPACBaseURL");
50
    unless ($url) {
51
        say "OPACBaseURL syspref isn't defined. You can use --url parameter.";
52
        exit;
53
    }
54
    $url = 'http://' . $url;
55
}
56
$url =~ s/\/*$//g;
57
58
my $sitemaper = Koha::Sitemapper->new( 
59
    verbose => $verbose,
60
    url     => $url,
61
    dir     => $dir,
62
    short   => $short,
63
);
64
$sitemaper->run(); 
65
66
67
=head1 USAGE
68
69
=over
70
71
=item sitemap.pl [--verbose|--help|--short|--noshort|--url|--dir]
72
73
=back
74
75
head 1 SYNOPSIS
76
77
=over
78
79
  sitemap.pl --verbose
80
  sitemap.pl --noshort --url /home/koha/mylibrary/www
81
82
=back
83
84
=head1 DESCRIPTION
85
86
Process all biblio records from a Koha instance and generate Sitemap files
87
complying with this protocol as described on L<http://sitemaps.org>. The goal of
88
this script is to be able to provide to search engines direct access to biblio
89
records. It avoid leaving search engine browsing Koha OPAC and so generating
90
a lot of traffic, and workload, for a bad result. 
91
92
A file name F<sitemapindex.xml> is generated. It contains references to Sitemap
93
multiples files. Each file contains at most 50,000 urls, and is named
94
F<sitemapXXXX.xml>.
95
96
The files must be stored on Koha OPAC root directoty, ie
97
F<<koha-root>/koha-tmpl/>. Place also in this directory a F<robots.txt> file
98
like this one:
99
100
 Sitemap: sitemapindex.xml
101
 User-agent: *
102
 Disallow: /cgi-bin/
103
104
=head1 PARAMETERS
105
106
=over
107
108
=item B<--url=Koha OPAC base URL>
109
110
If omitted, OPACBaseURL syspref is used.
111
112
=item B<--short|noshort>
113
114
By default, --short. With --short, URL to bib record ends with
115
/bib/biblionumber. With --noshort, URL ends with
116
/cgi-bin/koha/opac-detail.pl?biblionumber=
117
118
=item B<--dir>
119
120
Directory where to write sitemap files. By default, the current directory.
121
122
=item B<--verbose|-v>
123
124
Enable script verbose mode: each minute, the number of biblio record links
125
written is displayed.
126
127
=item B<--help|-h>
128
129
Print this help page.
130
131
=back
132
133
=cut
134
(-)a/t/Sitemapper.t (-1 / +11 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
#
3
use Modern::Perl;
4
5
use Test::More tests => 2;
6
7
BEGIN {
8
    use_ok('Koha::Sitemapper');
9
    use_ok('Koha::Sitemapper::Writer');
10
}
11

Return to bug 11190