View | Details | Raw Unified | Return to bug 11190
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
Lines 612-617 our $PERL_DEPS = { Link Here
612
        'required' => '0',
612
        'required' => '0',
613
        'min_ver'  => '2.13',
613
        'min_ver'  => '2.13',
614
    },
614
    },
615
    'Moo' => {
616
        'usage'    => 'Core',
617
        'required' => '0',
618
        'min_ver'  => '1',
619
    },
615
    'String::Random' => {
620
    'String::Random' => {
616
        'usage'    => 'OpacSelfRegistration',
621
        'usage'    => 'OpacSelfRegistration',
617
        'required' => '1',
622
        'required' => '1',
Lines 737-742 our $PERL_DEPS = { Link Here
737
        'required' => '0',
742
        'required' => '0',
738
        'min_ver'  => '5.61',
743
        'min_ver'  => '5.61',
739
    },
744
    },
745
    'XML::Writer' => {
746
        'usage'    => 'Command line scripts',
747
        'required' => '0',
748
        'min_ver'  => '0.614',
749
    },
740
};
750
};
741
751
742
1;
752
1;
(-)a/Koha/Sitemapper.pm (+89 lines)
Line 0 Link Here
1
package Koha::Sitemapper;
2
3
#
4
# Copyright 2015 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 3 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
use Moo;
22
use Modern::Perl;
23
use Koha::Sitemapper::Writer;
24
use C4::Context;
25
26
27
has url => ( is => 'rw', );
28
29
has dir => (
30
    is => 'rw',
31
    default => '.',
32
    trigger => sub {
33
        my ($self, $dir) = @_;
34
        unless (-d $dir) {
35
            say "This is not a valid directory: $dir";
36
            exit;
37
        }
38
    }
39
);
40
41
has short => ( is => 'rw', default => 1 );
42
43
has verbose => ( is => 'rw', default => 0 );
44
45
has sth => ( is => 'rw' );
46
47
has writer => ( is => 'rw', );
48
49
has count => ( is => 'rw', default => 0);
50
51
52
sub run {
53
    my $self = shift;
54
55
    say "Creation of Sitemap files in '" . $self->dir . "' directory"
56
        if $self->verbose;
57
58
    $self->writer( Koha::Sitemapper::Writer->new( sitemapper => $self ) );
59
    my $sth = C4::Context->dbh->prepare(
60
         "SELECT biblionumber, timestamp FROM biblio" );
61
    $sth->execute();
62
    $self->sth($sth);
63
64
    while ( $self->process() ) {
65
        say "..... ", $self->count
66
            if $self->verbose && $self->count % 10000 == 0;
67
    }
68
}
69
70
71
sub process {
72
    my $self = shift;
73
74
    my ($biblionumber, $timestamp) = $self->sth->fetchrow;
75
    unless ($biblionumber) {
76
        $self->writer->end();
77
        say "Number of biblio records processed: ", $self->count, "\n" .
78
            "Number of Sitemap files:            ", $self->writer->count
79
            if $self->verbose;
80
        return;
81
    }
82
83
    $self->writer->write($biblionumber, $timestamp);
84
    $self->count( $self->count + 1 );
85
    return $self->count;
86
}
87
88
89
1;
(-)a/Koha/Sitemapper/Writer.pm (+125 lines)
Line 0 Link Here
1
package Koha::Sitemapper::Writer;
2
3
#
4
# Copyright 2015 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 3 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
use Moo;
23
use Modern::Perl;
24
use XML::Writer;
25
use IO::File;
26
use DateTime;
27
28
29
my $MAX = 50000;
30
31
32
has sitemapper => (is => 'rw', );
33
34
has current => ( is => 'rw', default => $MAX );
35
36
has count => ( is => 'rw', default => 0 );
37
38
has writer => ( is => 'rw',  );
39
40
41
42
sub _writer_create {
43
    my ($self, $name) = @_;
44
    $name = $self->sitemapper->dir . "/$name";
45
    my $fh = IO::File->new(">$name");
46
    unless ($fh) {
47
        say "Impossible to create file: $name";
48
        exit;
49
    }
50
    my $writer = XML::Writer->new(
51
        OUTPUT => $fh,
52
        DATA_MODE => 1,
53
        DATA_INDENT => 2,
54
    );
55
    $writer->xmlDecl("UTF-8");
56
    return $writer;
57
}
58
59
60
sub _writer_end {
61
    my $self = shift;
62
    return unless $self->writer;
63
    $self->writer->endTag();
64
    $self->writer->end();
65
    $self->writer->getOutput()->close();
66
}
67
68
69
sub write {
70
    my ($self, $biblionumber, $timestamp) = @_;
71
72
    if ( $self->current == $MAX ) {
73
        $self->_writer_end();
74
        $self->count( $self->count + 1 );
75
        my $w = $self->_writer_create( sprintf("sitemap%04d.xml", $self->count) );
76
        $w->startTag(
77
            'urlset',
78
            'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
79
            'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
80
            'xsi:schemaLocation' => 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd');
81
        $self->writer($w);
82
        $self->current(0);
83
    }
84
85
    $self->current( $self->current + 1 );
86
    my $writer = $self->writer;
87
    my $url = $self->sitemapper->url .
88
              ($self->sitemapper->short ? '/bib/' : '/cgi-bin/koha/opac-detail.pl?biblionumber=') .
89
              $biblionumber;
90
    $writer->startTag('url');
91
        $writer->startTag('loc');
92
            $writer->characters($url);
93
        $writer->endTag();
94
        $writer->startTag('lastmod');
95
            $timestamp = substr($timestamp, 0, 10);
96
            $writer->characters($timestamp);
97
        $writer->endTag();
98
    $writer->endTag();
99
}
100
101
102
sub end {
103
    my $self = shift;
104
105
    $self->_writer_end();
106
107
    my $w = $self->_writer_create("sitemapindex.xml");
108
    $w->startTag('sitemapindex', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9');
109
    my $now = DateTime->now()->ymd;
110
    for my $i ( 1..$self->count ) {
111
        $w->startTag('sitemap');
112
            $w->startTag('loc');
113
                my $name = sprintf("sitemap%04d.xml", $i);
114
                $w->characters($self->sitemapper->url . "/$name");
115
            $w->endTag();
116
            $w->startTag('lastmod');
117
                $w->characters($now);
118
            $w->endTag();
119
        $w->endTag();
120
    }
121
    $w->endTag();
122
}
123
124
125
1;
(-)a/misc/cronjobs/sitemap.pl (+127 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2015 Tamil s.a.r.l.
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
package Main;
21
22
use Modern::Perl;
23
use utf8;
24
use Pod::Usage;
25
use Getopt::Long;
26
use C4::Biblio;
27
use Koha::Sitemapper;
28
29
30
my ($verbose, $help, $url, $dir, $short) = (0, 0, '', '.', 1);
31
GetOptions(
32
    'verbose'   => \$verbose,
33
    'help'      => \$help,
34
    'url=s'     => \$url,
35
    'dir=s'     => \$dir,
36
    'short!'    => \$short,
37
);
38
39
sub usage {
40
    pod2usage( -verbose => 2 );
41
    exit;
42
}
43
44
usage() if $help;
45
46
unless ($url) {
47
    $url = C4::Context->preference("OPACBaseURL");
48
    unless ($url) {
49
        say "OPACBaseURL syspref isn't defined. You can use --url parameter.";
50
        exit;
51
    }
52
    $url = 'http://' . $url;
53
}
54
$url =~ s/\/*$//g;
55
56
my $sitemaper = Koha::Sitemapper->new(
57
    verbose => $verbose,
58
    url     => $url,
59
    dir     => $dir,
60
    short   => $short,
61
);
62
$sitemaper->run();
63
64
65
=head1 USAGE
66
67
=over
68
69
=item sitemap.pl [--verbose|--help|--short|--noshort|--url|--dir]
70
71
=back
72
73
=head1 SYNOPSIS
74
75
  sitemap.pl --verbose
76
  sitemap.pl --noshort --url /home/koha/mylibrary/www
77
78
=head1 DESCRIPTION
79
80
Process all biblio records from a Koha instance and generate Sitemap files
81
complying with this protocol as described on L<http://sitemaps.org>. The goal of
82
this script is to be able to provide to search engines direct access to biblio
83
records. It avoid leaving search engine browsing Koha OPAC and so generating
84
a lot of traffic, and workload, for a bad result.
85
86
A file name F<sitemapindex.xml> is generated. It contains references to Sitemap
87
multiples files. Each file contains at most 50,000 urls, and is named
88
F<sitemapXXXX.xml>.
89
90
The files must be stored on Koha OPAC root directory, ie
91
F<<koha-root>/koha-tmpl/>. Place also in this directory a F<robots.txt> file
92
like this one:
93
94
 Sitemap: sitemapindex.xml
95
 User-agent: *
96
 Disallow: /cgi-bin/
97
98
=head1 PARAMETERS
99
100
=over
101
102
=item B<--url=Koha OPAC base URL>
103
104
If omitted, OPACBaseURL syspref is used.
105
106
=item B<--short|noshort>
107
108
By default, --short. With --short, URL to bib record ends with
109
/bib/biblionumber. With --noshort, URL ends with
110
/cgi-bin/koha/opac-detail.pl?biblionumber=bibnum
111
112
=item B<--dir>
113
114
Directory where to write sitemap files. By default, the current directory.
115
116
=item B<--verbose|-v>
117
118
Enable script verbose mode: a message is displayed for each 10,000 biblio
119
records processed.
120
121
=item B<--help|-h>
122
123
Print this help page.
124
125
=back
126
127
=cut
(-)a/t/db_dependent/Sitemapper.t (-1 / +193 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright 2015 Tamil s.a.r.l.
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
use Modern::Perl;
21
use Test::MockModule;
22
use File::Basename;
23
use File::Path;
24
use DateTime;
25
use Test::More tests => 14;
26
27
28
BEGIN {
29
    use_ok('Koha::Sitemapper');
30
    use_ok('Koha::Sitemapper::Writer');
31
}
32
33
34
sub slurp {
35
    my $file = shift;
36
    open my $fh, '<', $file or die;
37
    local $/ = undef;
38
    my $cont = <$fh>;
39
    close $fh;
40
    return $cont;
41
}
42
43
44
# Create 3 mocked dataset to be used by Koha::Sitemaper in place of DB content
45
my $module_context = new Test::MockModule('C4::Context');
46
$module_context->mock('_new_dbh', sub {
47
    my $dbh = DBI->connect( 'DBI:Mock:', '', '' )
48
    || die "Cannot create handle: $DBI::errstr\n";
49
    return $dbh
50
});
51
my $dbh = C4::Context->dbh();
52
my $two_bibs = [
53
	[ qw/ biblionumber timestamp  / ],
54
	[ qw/ 1234         2013-11-15 / ],
55
	[ qw/ 9875         2015-08-31 / ],	
56
];
57
my $lotof_bibs = [ [ qw/ biblionumber timestamp / ] ];
58
push @$lotof_bibs, [ $_, '2015-08-31' ] for 1..75000;
59
$dbh->{mock_add_resultset} = $two_bibs;
60
$dbh->{mock_add_resultset} = $two_bibs;
61
$dbh->{mock_add_resultset} = $lotof_bibs;
62
63
my $dir = File::Spec->rel2abs( dirname(__FILE__) );
64
65
# Create a sitemap for a catalog containg 2 biblios, with option 'long url'
66
my $sitemaper = Koha::Sitemapper->new(
67
    verbose => 0,
68
    url     => 'http://www.mylibrary.org',
69
    dir     => $dir,
70
    short   => 0,
71
);
72
$sitemaper->run();
73
74
my $file = "$dir/sitemapindex.xml";
75
ok( -e "$dir/sitemapindex.xml", "File sitemapindex.xml created");
76
my $file_content = slurp($file);
77
my $now = DateTime->now->ymd;
78
my $expected_content = <<EOS;
79
<?xml version="1.0" encoding="UTF-8"?>
80
81
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
82
  <sitemap>
83
    <loc>http://www.mylibrary.org/sitemap0001.xml</loc>
84
    <lastmod>$now</lastmod>
85
  </sitemap>
86
</sitemapindex>
87
EOS
88
chop $expected_content;
89
ok( $file_content eq $expected_content, "Its content is valid" );
90
91
$file = "$dir/sitemap0001.xml";
92
ok( -e $file, "File sitemap0001.xml created");
93
$file_content = slurp($file);
94
$expected_content = <<EOS;
95
<?xml version="1.0" encoding="UTF-8"?>
96
97
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
98
  <url>
99
    <loc>http://www.mylibrary.org/cgi-bin/koha/opac-detail.pl?biblionumber=1234</loc>
100
    <lastmod>2013-11-15</lastmod>
101
  </url>
102
  <url>
103
    <loc>http://www.mylibrary.org/cgi-bin/koha/opac-detail.pl?biblionumber=9875</loc>
104
    <lastmod>2015-08-31</lastmod>
105
  </url>
106
</urlset>
107
EOS
108
ok( $file_content eq $expected_content, "Its content is valid" );
109
110
111
# Create a sitemap for a catalog containg 2 biblios, with option 'short url'.
112
# Test that 2 files are created.
113
$sitemaper = Koha::Sitemapper->new(
114
    verbose => 0,
115
    url     => 'http://www.mylibrary.org',
116
    dir     => $dir,
117
    short   => 1,
118
);
119
$sitemaper->run();
120
121
$file = "$dir/sitemap0001.xml";
122
ok( -e $file, "File sitemap0001.xml with short URLs created");
123
$file_content = slurp($file);
124
$expected_content = <<EOS;
125
<?xml version="1.0" encoding="UTF-8"?>
126
127
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
128
  <url>
129
    <loc>http://www.mylibrary.org/bib/1234</loc>
130
    <lastmod>2013-11-15</lastmod>
131
  </url>
132
  <url>
133
    <loc>http://www.mylibrary.org/bib/9875</loc>
134
    <lastmod>2015-08-31</lastmod>
135
  </url>
136
</urlset>
137
EOS
138
ok( $file_content eq $expected_content, "Its content is valid" );
139
140
141
# Create a sitemap for a catalog containing 75000 biblios, with option 'short
142
# url'. Test that 3 files are created: index file + 2 urls file with
143
# respectively 50000 et 25000 urls.
144
$sitemaper = Koha::Sitemapper->new(
145
    verbose => 0,
146
    url     => 'http://www.mylibrary.org',
147
    dir     => $dir,
148
    short   => 1,
149
);
150
$sitemaper->run();
151
152
$file = "$dir/sitemapindex.xml";
153
ok( -e "$dir/sitemapindex.xml", "File sitemapindex.xml for 75000 bibs created");
154
$file_content = slurp($file);
155
$expected_content = <<EOS;
156
<?xml version="1.0" encoding="UTF-8"?>
157
158
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
159
  <sitemap>
160
    <loc>http://www.mylibrary.org/sitemap0001.xml</loc>
161
    <lastmod>$now</lastmod>
162
  </sitemap>
163
  <sitemap>
164
    <loc>http://www.mylibrary.org/sitemap0002.xml</loc>
165
    <lastmod>$now</lastmod>
166
  </sitemap>
167
</sitemapindex>
168
EOS
169
chop $expected_content;
170
ok( $file_content eq $expected_content, "Its content is valid" );
171
172
$file = "$dir/sitemap0001.xml";
173
ok( -e $file, "File sitemap0001.xml created");
174
175
open my $fh, "<", $file;
176
my $count = 0;
177
while (<$fh>) {
178
	$count++ if /<loc>/;
179
}
180
ok ( $count == 50000, "It contains 50000 URLs");
181
182
$file = "$dir/sitemap0002.xml";
183
ok( -e $file, "File sitemap0002.xml created");
184
185
open $fh, "<", $file;
186
$count = 0;
187
while (<$fh>) {
188
	$count++ if /<loc>/;
189
}
190
ok ( $count == 25000, "It contains 25000 URLs");
191
192
# Cleanup
193
unlink "$dir/$_" for qw / sitemapindex.xml sitemap0001.xml sitemap0002.xml /;

Return to bug 11190