View | Details | Raw Unified | Return to bug 11190
Collapse All | Expand All

(-)a/Koha/Sitemaper.pm (+82 lines)
Line 0 Link Here
1
package Koha::Sitemaper;
2
3
#
4
# Copyright 2013 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
use Moose;
22
23
extends 'AnyEvent::Processor';
24
25
use Modern::Perl;
26
use Koha::Sitemaper::Writer;
27
use C4::Context;
28
29
30
has url => ( is => 'rw', isa => 'Str' );
31
32
has verbose => ( is => 'rw', isa => 'Bool', default => 0 );
33
34
has sth => ( is => 'rw' );
35
36
has writer => ( is => 'rw', isa => 'Koha::Sitemaper::Writer' );
37
38
39
before 'run' => sub {
40
    my $self = shift;
41
42
    $self->writer( Koha::Sitemaper::Writer->new( url => $self->url ) );
43
44
    my $sth = C4::Context->dbh->prepare(
45
         "SELECT biblionumber, timestamp FROM biblio" );
46
    $sth->execute();
47
    $self->sth( $sth );
48
};
49
50
51
override 'process' => sub {
52
    my $self = shift;
53
54
    my ($biblionumber, $timestamp) = $self->sth->fetchrow;
55
    return unless $biblionumber;
56
57
    $self->writer->write($biblionumber, $timestamp);
58
    return super();
59
};
60
61
62
before 'end_process' => sub { shift->writer->end(); };
63
64
65
override 'start_message' => sub {
66
    say "Creation of Sitemap files";
67
};
68
69
70
override 'end_message' => sub {
71
    my $self = shift;
72
    say "Number of biblio records processed: ", $self->count, "\n" .
73
        "Number of Sitemap files:            ", $self->writer->count;
74
};
75
76
77
no Moose;
78
__PACKAGE__->meta->make_immutable;
79
80
81
1;
82
(-)a/Koha/Sitemaper/Writer.pm (+112 lines)
Line 0 Link Here
1
package Koha::Sitemaper::Writer;
2
3
#
4
# Copyright 2013 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
use Moose;
23
use Modern::Perl;
24
use XML::Writer;
25
use IO::File;
26
use DateTime;
27
28
29
my $MAX = 50000;
30
31
has url => ( is => 'rw', isa => 'Str');
32
33
has current => ( is => 'rw', isa => 'Int', default => $MAX );
34
35
has count => ( is => 'rw', isa => 'Int', default => 0 );
36
37
has writer => ( is => 'rw', isa => 'XML::Writer' );
38
39
40
41
sub _writer_create {
42
    my ($self, $name) = @_;
43
    my $fh = IO::File->new(">$name");
44
    my $writer = XML::Writer->new(
45
        OUTPUT => $fh,
46
        DATA_MODE => 1,
47
        DATA_INDENT => 2,
48
    );
49
    return $writer;
50
}
51
52
53
sub _writer_end {
54
    my $self = shift;
55
    return unless $self->writer;
56
    $self->writer->endTag();
57
    $self->writer->end();
58
}
59
60
61
sub write {
62
    my ($self, $biblionumber, $timestamp) = @_;
63
64
    if ( $self->current == $MAX ) {
65
        $self->_writer_end();
66
        $self->count( $self->count + 1 );
67
        my $w = $self->_writer_create( sprintf("sitemap%04d.xml", $self->count) );
68
        $w->startTag('urlset', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9');
69
        $self->writer($w);
70
        $self->current(0);
71
    }
72
73
    $self->current( $self->current + 1 );
74
    my $writer = $self->writer;
75
    $writer->startTag('url');
76
        $writer->startTag('loc');
77
            $writer->characters($self->url . "/bib/$biblionumber");
78
        $writer->endTag();
79
        $writer->startTag('lastmod');
80
            $timestamp = substr($timestamp, 0, 10);
81
            $writer->characters($timestamp);
82
        $writer->endTag();
83
    $writer->endTag();
84
}
85
86
87
sub end {
88
    my $self = shift;
89
90
    $self->_writer_end();
91
92
    my $w = $self->_writer_create("sitemapindex.xml");
93
    $w->startTag('sitemapindex', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9');
94
    my $now = DateTime->now()->ymd;
95
    for my $i ( 1..$self->count ) {
96
        $w->startTag('sitemap');
97
            $w->startTag('loc');
98
                my $name = sprintf("sitemap%04d.xml", $i);
99
                $w->characters($self->url . "/$name");
100
            $w->endTag();
101
            $w->startTag('lastmod');
102
                $w->characters($now);
103
            $w->endTag();
104
        $w->endTag();
105
    }
106
    $w->endTag();
107
}
108
109
110
no Moose;
111
1;
112
(-)a/misc/cronjobs/sitemap.pl (-1 / +98 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
#
4
# Copyright 2013 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
package Main;
23
24
use Modern::Perl;
25
use utf8;
26
use Pod::Usage;
27
use Getopt::Long;
28
use Koha::Sitemaper;
29
30
31
my ($verbose, $help, $url) = (0, 0, '');
32
GetOptions( 
33
    'verbose'   => \$verbose,
34
    'help'      => \$help,
35
    'url=s'     => \$url,
36
);
37
38
sub usage {
39
    pod2usage( -verbose => 2 );
40
    exit;
41
} 
42
43
44
usage() if $help || !$url;          
45
46
my $sitemaper = Koha::Sitemaper->new( 
47
    verbose => $verbose,
48
    url     => $url,
49
);
50
$sitemaper->run(); 
51
52
53
=head1 USAGE
54
55
=over
56
57
=item sitemap.pl [--verbose|--help] --url=F<Koha OPAC base URL> 
58
59
=back
60
61
=head1 DESCRIPTION
62
63
Process all biblio records from a Koha instance and generate Sitemap files
64
complying with this protocol as described on L<http://sitemaps.org>. The goal of
65
this script is to be able to provide to search engines direct access to biblio
66
records. It avoid leaving search engine browsing Koha OPAC and so generating
67
a lot of traffic, and workload, for a bad result. 
68
69
A file name F<sitemapindex.xml> is generated. It contains references to Sitemap
70
multiples files. Each file contains at most 50,000 urls, and is named
71
F<sitemapXXXX.xml>.
72
73
The files must be stored on Koha OPAC root directoty, ie
74
F<<koha-root>/koha-tmpl/>. Place also in this directory a F<robots.txt> file
75
like this one:
76
77
 Sitemap: sitemapindex.xml
78
 User-agent: *
79
 Disallow: /cgi-bin/
80
81
=head1 PARAMETERS
82
83
=over
84
85
=item B<--url=Koha OPAC base URL>
86
87
=item B<--verbose|-v>
88
89
Enable script verbose mode. 
90
91
=item B<--help|-h>
92
93
Print this help page.
94
95
=back
96
97
=cut
98

Return to bug 11190