View | Details | Raw Unified | Return to bug 11190
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (+10 lines)
Lines 629-634 our $PERL_DEPS = { Link Here
629
        'required' => '0',
629
        'required' => '0',
630
        'min_ver'  => '2.13',
630
        'min_ver'  => '2.13',
631
    },
631
    },
632
    'AnyEvent::HTTP' => {
633
        'usage'    => 'Command line scripts',
634
        'required' => '0',
635
        'min_ver'  => '0.003',
636
    },
632
    'Moose' => {
637
    'Moose' => {
633
        'usage'    => 'Core',
638
        'usage'    => 'Core',
634
        'required' => '0',
639
        'required' => '0',
Lines 704-709 our $PERL_DEPS = { Link Here
704
        'required' => '0',
709
        'required' => '0',
705
        'min_ver'  => '0.73',
710
        'min_ver'  => '0.73',
706
    },
711
    },
712
    'XML::Writer' => {
713
        'usage'    => 'Command line scripts',
714
        'required' => '0',
715
        'min_ver'  => '0.614',
716
    },
707
};
717
};
708
718
709
1;
719
1;
(-)a/Koha/Sitemapper.pm (+85 lines)
Line 0 Link Here
1
package Koha::Sitemapper;
2
3
#
4
# Copyright 2013 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
use Moose;
22
23
extends 'AnyEvent::Processor';
24
25
use Modern::Perl;
26
use Koha::Sitemapper::Writer;
27
use C4::Context;
28
29
30
has url => ( is => 'rw', isa => 'Str' );
31
32
has short => ( is => 'rw', isa => 'Bool', default => 1 );
33
34
has verbose => ( is => 'rw', isa => 'Bool', default => 0 );
35
36
has sth => ( is => 'rw' );
37
38
has writer => ( is => 'rw', isa => 'Koha::Sitemapper::Writer' );
39
40
41
before 'run' => sub {
42
    my $self = shift;
43
44
    $self->writer( Koha::Sitemapper::Writer->new(
45
        url => $self->url, short => $self->short ) );
46
47
    my $sth = C4::Context->dbh->prepare(
48
         "SELECT biblionumber, timestamp FROM biblio" );
49
    $sth->execute();
50
    $self->sth( $sth );
51
};
52
53
54
override 'process' => sub {
55
    my $self = shift;
56
57
    my ($biblionumber, $timestamp) = $self->sth->fetchrow;
58
    return unless $biblionumber;
59
60
    $self->writer->write($biblionumber, $timestamp);
61
    return super();
62
};
63
64
65
before 'end_process' => sub { shift->writer->end(); };
66
67
68
override 'start_message' => sub {
69
    say "Creation of Sitemap files";
70
};
71
72
73
override 'end_message' => sub {
74
    my $self = shift;
75
    say "Number of biblio records processed: ", $self->count, "\n" .
76
        "Number of Sitemap files:            ", $self->writer->count;
77
};
78
79
80
no Moose;
81
__PACKAGE__->meta->make_immutable;
82
83
84
1;
85
(-)a/Koha/Sitemapper/Writer.pm (+122 lines)
Line 0 Link Here
1
package Koha::Sitemapper::Writer;
2
3
#
4
# Copyright 2014 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
use Moose;
23
use Modern::Perl;
24
use XML::Writer;
25
use IO::File;
26
use DateTime;
27
28
29
my $MAX = 50000;
30
31
has url => ( is => 'rw', isa => 'Str');
32
33
has short => ( is => 'rw', isa => 'Bool', default => 1);
34
35
has current => ( is => 'rw', isa => 'Int', default => $MAX );
36
37
has count => ( is => 'rw', isa => 'Int', default => 0 );
38
39
has writer => ( is => 'rw', isa => 'XML::Writer' );
40
41
42
43
sub _writer_create {
44
    my ($self, $name) = @_;
45
    my $fh = IO::File->new(">$name");
46
    my $writer = XML::Writer->new(
47
        OUTPUT => $fh,
48
        DATA_MODE => 1,
49
        DATA_INDENT => 2,
50
    );
51
    $writer->xmlDecl("UTF-8");
52
    return $writer;
53
}
54
55
56
sub _writer_end {
57
    my $self = shift;
58
    return unless $self->writer;
59
    $self->writer->endTag();
60
    $self->writer->end();
61
}
62
63
64
sub write {
65
    my ($self, $biblionumber, $timestamp) = @_;
66
67
    if ( $self->current == $MAX ) {
68
        $self->_writer_end();
69
        $self->count( $self->count + 1 );
70
        my $w = $self->_writer_create( sprintf("sitemap%04d.xml", $self->count) );
71
        $w->startTag(
72
            'urlset',
73
            'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
74
            'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
75
            'xsi:schemaLocation' => 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd');
76
        $self->writer($w);
77
        $self->current(0);
78
    }
79
80
    $self->current( $self->current + 1 );
81
    my $writer = $self->writer;
82
    my $url = $self->url .
83
              ($self->short ? '/bib/' : '/cgi-bin/koha/opac-detail.pl?biblionumber=') .
84
              $biblionumber;
85
    $writer->startTag('url');
86
        $writer->startTag('loc');
87
            $writer->characters($url);
88
        $writer->endTag();
89
        $writer->startTag('lastmod');
90
            $timestamp = substr($timestamp, 0, 10);
91
            $writer->characters($timestamp);
92
        $writer->endTag();
93
    $writer->endTag();
94
}
95
96
97
sub end {
98
    my $self = shift;
99
100
    $self->_writer_end();
101
102
    my $w = $self->_writer_create("sitemapindex.xml");
103
    $w->startTag('sitemapindex', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9');
104
    my $now = DateTime->now()->ymd;
105
    for my $i ( 1..$self->count ) {
106
        $w->startTag('sitemap');
107
            $w->startTag('loc');
108
                my $name = sprintf("sitemap%04d.xml", $i);
109
                $w->characters($self->url . "/$name");
110
            $w->endTag();
111
            $w->startTag('lastmod');
112
                $w->characters($now);
113
            $w->endTag();
114
        $w->endTag();
115
    }
116
    $w->endTag();
117
}
118
119
120
no Moose;
121
1;
122
(-)a/misc/cronjobs/sitemap.pl (+119 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
#
4
# Copyright 2014 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 2 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
package Main;
23
24
use Modern::Perl;
25
use utf8;
26
use Pod::Usage;
27
use Getopt::Long;
28
use C4::Biblio;
29
use Koha::Sitemapper;
30
31
32
my ($verbose, $help, $url, $short) = (0, 0, '', 1);
33
GetOptions( 
34
    'verbose'   => \$verbose,
35
    'help'      => \$help,
36
    'url=s'     => \$url,
37
    'short!'    => \$short,
38
);
39
40
sub usage {
41
    pod2usage( -verbose => 2 );
42
    exit;
43
} 
44
45
usage() if $help;
46
47
unless ($url) {
48
    $url = C4::Context->preference("OPACBaseURL");
49
    unless ($url) {
50
        say "OPACBaseURL syspref isn't defined. You can use --url parameter.";
51
        exit;
52
    }
53
    $url = 'http://' . $url;
54
}
55
$url =~ s/\/*$//g;
56
57
my $sitemaper = Koha::Sitemapper->new( 
58
    verbose => $verbose,
59
    url     => $url,
60
    short   => $short,
61
);
62
$sitemaper->run(); 
63
64
65
=head1 USAGE
66
67
=over
68
69
=item sitemap.pl [--verbose|--help|short|noshort] --url=F<Koha OPAC base URL> 
70
71
=back
72
73
=head1 DESCRIPTION
74
75
Process all biblio records from a Koha instance and generate Sitemap files
76
complying with this protocol as described on L<http://sitemaps.org>. The goal of
77
this script is to be able to provide to search engines direct access to biblio
78
records. It avoid leaving search engine browsing Koha OPAC and so generating
79
a lot of traffic, and workload, for a bad result. 
80
81
A file name F<sitemapindex.xml> is generated. It contains references to Sitemap
82
multiples files. Each file contains at most 50,000 urls, and is named
83
F<sitemapXXXX.xml>.
84
85
The files must be stored on Koha OPAC root directoty, ie
86
F<<koha-root>/koha-tmpl/>. Place also in this directory a F<robots.txt> file
87
like this one:
88
89
 Sitemap: sitemapindex.xml
90
 User-agent: *
91
 Disallow: /cgi-bin/
92
93
=head1 PARAMETERS
94
95
=over
96
97
=item B<--url=Koha OPAC base URL>
98
99
If omitted, OPACBaseURL syspref is used.
100
101
=item B<--short|noshort>
102
103
By default, --short. With --short, URL to bib record ends with
104
/bib/biblionumber. With --noshort, URL ends with
105
/cgi-bin/koha/opac-detail.pl?biblionumber=
106
107
=item B<--verbose|-v>
108
109
Enable script verbose mode: each minute, the number of biblio record links
110
written is displayed.
111
112
=item B<--help|-h>
113
114
Print this help page.
115
116
=back
117
118
=cut
119
(-)a/t/Sitemapper.t (-1 / +11 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
#
3
use Modern::Perl;
4
5
use Test::More tests => 2;
6
7
BEGIN {
8
    use_ok('Koha::Sitemapper');
9
    use_ok('Koha::Sitemapper::Writer');
10
}
11

Return to bug 11190