View | Details | Raw Unified | Return to bug 11190
Collapse All | Expand All

(-)a/C4/Installer/PerlDependencies.pm (-2 / +12 lines)
Lines 607-616 our $PERL_DEPS = { Link Here
607
        'required' => '0',
607
        'required' => '0',
608
        'min_ver'  => '5.0',
608
        'min_ver'  => '5.0',
609
    },
609
    },
610
    'AnyEvent::HTTP' => {
610
    'AnyEvent::Processor' => {
611
        'usage'    => 'Command line scripts',
611
        'usage'    => 'Command line scripts',
612
        'required' => '0',
612
        'required' => '0',
613
        'min_ver'  => '2.13',
613
        'min_ver'  => '0.003',
614
    },
615
    'Moose' => {
616
        'usage'    => 'Core',
617
        'required' => '0',
618
        'min_ver'  => '1.09',
614
    },
619
    },
615
    'String::Random' => {
620
    'String::Random' => {
616
        'usage'    => 'OpacSelfRegistration',
621
        'usage'    => 'OpacSelfRegistration',
Lines 737-742 our $PERL_DEPS = { Link Here
737
        'required' => '0',
742
        'required' => '0',
738
        'min_ver'  => '5.61',
743
        'min_ver'  => '5.61',
739
    },
744
    },
745
    'XML::Writer' => {
746
        'usage'    => 'Command line scripts',
747
        'required' => '0',
748
        'min_ver'  => '0.614',
749
    },
740
};
750
};
741
751
742
1;
752
1;
(-)a/Koha/Sitemapper.pm (+95 lines)
Line 0 Link Here
1
package Koha::Sitemapper;
2
3
#
4
# Copyright 2013 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 3 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
use Moose;
22
23
extends 'AnyEvent::Processor';
24
25
use Modern::Perl;
26
use Koha::Sitemapper::Writer;
27
use C4::Context;
28
29
30
has url => ( is => 'rw', isa => 'Str' );
31
32
has dir => (
33
    is => 'rw',
34
    isa => 'Str',
35
    default => '.',
36
    trigger => sub {
37
        my ($self, $dir) = @_;
38
        unless (-d $dir) {
39
            say "This is not a valid directory: $dir";
40
            exit;
41
        }
42
    }
43
);
44
45
has short => ( is => 'rw', isa => 'Bool', default => 1 );
46
47
has verbose => ( is => 'rw', isa => 'Bool', default => 0 );
48
49
has sth => ( is => 'rw' );
50
51
has writer => ( is => 'rw', isa => 'Koha::Sitemapper::Writer' );
52
53
54
before 'run' => sub {
55
    my $self = shift;
56
57
    $self->writer( Koha::Sitemapper::Writer->new( sitemapper => $self ) );
58
    my $sth = C4::Context->dbh->prepare(
59
         "SELECT biblionumber, timestamp FROM biblio" );
60
    $sth->execute();
61
    $self->sth( $sth );
62
};
63
64
65
override 'process' => sub {
66
    my $self = shift;
67
68
    my ($biblionumber, $timestamp) = $self->sth->fetchrow;
69
    return unless $biblionumber;
70
71
    $self->writer->write($biblionumber, $timestamp);
72
    return super();
73
};
74
75
76
before 'end_process' => sub { shift->writer->end(); };
77
78
79
override 'start_message' => sub {
80
    say "Creation of Sitemap files in '" . shift->dir . "' directory";
81
};
82
83
84
override 'end_message' => sub {
85
    my $self = shift;
86
    say "Number of biblio records processed: ", $self->count, "\n" .
87
        "Number of Sitemap files:            ", $self->writer->count;
88
};
89
90
91
no Moose;
92
__PACKAGE__->meta->make_immutable;
93
94
95
1;
(-)a/Koha/Sitemapper/Writer.pm (+125 lines)
Line 0 Link Here
1
package Koha::Sitemapper::Writer;
2
3
#
4
# Copyright 2014 Tamil s.a.r.l.
5
#
6
# This file is part of Koha.
7
#
8
# Koha is free software; you can redistribute it and/or modify it under the
9
# terms of the GNU General Public License as published by the Free Software
10
# Foundation; either version 3 of the License, or (at your option) any later
11
# version.
12
#
13
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
14
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
16
#
17
# You should have received a copy of the GNU General Public License along
18
# with Koha; if not, write to the Free Software Foundation, Inc.,
19
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20
21
22
use Moose;
23
use Modern::Perl;
24
use XML::Writer;
25
use IO::File;
26
use DateTime;
27
28
29
my $MAX = 50000;
30
31
32
has sitemapper => (is => 'rw', isa => 'Koha::Sitemapper');
33
34
has current => ( is => 'rw', isa => 'Int', default => $MAX );
35
36
has count => ( is => 'rw', isa => 'Int', default => 0 );
37
38
has writer => ( is => 'rw', isa => 'XML::Writer' );
39
40
41
42
sub _writer_create {
43
    my ($self, $name) = @_;
44
    $name = $self->sitemapper->dir . "/$name";
45
    my $fh = IO::File->new(">$name");
46
    unless ($fh) {
47
        say "Impossible to create file: $name";
48
        exit;
49
    }
50
    my $writer = XML::Writer->new(
51
        OUTPUT => $fh,
52
        DATA_MODE => 1,
53
        DATA_INDENT => 2,
54
    );
55
    $writer->xmlDecl("UTF-8");
56
    return $writer;
57
}
58
59
60
sub _writer_end {
61
    my $self = shift;
62
    return unless $self->writer;
63
    $self->writer->endTag();
64
    $self->writer->end();
65
}
66
67
68
sub write {
69
    my ($self, $biblionumber, $timestamp) = @_;
70
71
    if ( $self->current == $MAX ) {
72
        $self->_writer_end();
73
        $self->count( $self->count + 1 );
74
        my $w = $self->_writer_create( sprintf("sitemap%04d.xml", $self->count) );
75
        $w->startTag(
76
            'urlset',
77
            'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9',
78
            'xmlns:xsi' => 'http://www.w3.org/2001/XMLSchema-instance',
79
            'xsi:schemaLocation' => 'http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd');
80
        $self->writer($w);
81
        $self->current(0);
82
    }
83
84
    $self->current( $self->current + 1 );
85
    my $writer = $self->writer;
86
    my $url = $self->sitemapper->url .
87
              ($self->sitemapper->short ? '/bib/' : '/cgi-bin/koha/opac-detail.pl?biblionumber=') .
88
              $biblionumber;
89
    $writer->startTag('url');
90
        $writer->startTag('loc');
91
            $writer->characters($url);
92
        $writer->endTag();
93
        $writer->startTag('lastmod');
94
            $timestamp = substr($timestamp, 0, 10);
95
            $writer->characters($timestamp);
96
        $writer->endTag();
97
    $writer->endTag();
98
}
99
100
101
sub end {
102
    my $self = shift;
103
104
    $self->_writer_end();
105
106
    my $w = $self->_writer_create("sitemapindex.xml");
107
    $w->startTag('sitemapindex', 'xmlns' => 'http://www.sitemaps.org/schemas/sitemap/0.9');
108
    my $now = DateTime->now()->ymd;
109
    for my $i ( 1..$self->count ) {
110
        $w->startTag('sitemap');
111
            $w->startTag('loc');
112
                my $name = sprintf("sitemap%04d.xml", $i);
113
                $w->characters($self->sitemapper->url . "/$name");
114
            $w->endTag();
115
            $w->startTag('lastmod');
116
                $w->characters($now);
117
            $w->endTag();
118
        $w->endTag();
119
    }
120
    $w->endTag();
121
}
122
123
124
no Moose;
125
1;
(-)a/misc/cronjobs/sitemap.pl (+128 lines)
Line 0 Link Here
1
#!/usr/bin/perl
2
3
# Copyright 2014 Tamil s.a.r.l.
4
#
5
# This file is part of Koha.
6
#
7
# Koha is free software; you can redistribute it and/or modify it under the
8
# terms of the GNU General Public License as published by the Free Software
9
# Foundation; either version 3 of the License, or (at your option) any later
10
# version.
11
#
12
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License along
17
# with Koha; if not, write to the Free Software Foundation, Inc.,
18
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
package Main;
21
22
use Modern::Perl;
23
use utf8;
24
use Pod::Usage;
25
use Getopt::Long;
26
use C4::Biblio;
27
use Koha::Sitemapper;
28
29
30
my ($verbose, $help, $url, $dir, $short) = (0, 0, '', '.', 1);
31
GetOptions(
32
    'verbose'   => \$verbose,
33
    'help'      => \$help,
34
    'url=s'     => \$url,
35
    'dir=s'     => \$dir,
36
    'short!'    => \$short,
37
);
38
39
sub usage {
40
    pod2usage( -verbose => 2 );
41
    exit;
42
}
43
44
usage() if $help;
45
46
unless ($url) {
47
    $url = C4::Context->preference("OPACBaseURL");
48
    unless ($url) {
49
        say "OPACBaseURL syspref isn't defined. You can use --url parameter.";
50
        exit;
51
    }
52
    $url = 'http://' . $url;
53
}
54
$url =~ s/\/*$//g;
55
56
my $sitemaper = Koha::Sitemapper->new(
57
    verbose => $verbose,
58
    url     => $url,
59
    dir     => $dir,
60
    short   => $short,
61
);
62
$sitemaper->run();
63
64
65
=head1 USAGE
66
67
=over
68
69
=item sitemap.pl [--verbose|--help|--short|--noshort|--url|--dir]
70
71
=back
72
73
=head1 SYNOPSIS
74
75
  sitemap.pl --verbose
76
  sitemap.pl --noshort --url /home/koha/mylibrary/www
77
78
=head1 DESCRIPTION
79
80
Process all biblio records from a Koha instance and generate Sitemap files
81
complying with this protocol as described on L<http://sitemaps.org>. The goal of
82
this script is to be able to provide to search engines direct access to biblio
83
records. It avoid leaving search engine browsing Koha OPAC and so generating
84
a lot of traffic, and workload, for a bad result.
85
86
A file name F<sitemapindex.xml> is generated. It contains references to Sitemap
87
multiples files. Each file contains at most 50,000 urls, and is named
88
F<sitemapXXXX.xml>.
89
90
The files must be stored on Koha OPAC root directory, ie
91
F<<koha-root>/koha-tmpl/>. Place also in this directory a F<robots.txt> file
92
like this one:
93
94
 Sitemap: sitemapindex.xml
95
 User-agent: *
96
 Disallow: /cgi-bin/
97
98
=head1 PARAMETERS
99
100
=over
101
102
=item B<--url=Koha OPAC base URL>
103
104
If omitted, OPACBaseURL syspref is used.
105
106
=item B<--short|noshort>
107
108
By default, --short. With --short, URL to bib record ends with
109
/bib/biblionumber. With --noshort, URL ends with
110
/cgi-bin/koha/opac-detail.pl?biblionumber=bibnum
111
112
=item B<--dir>
113
114
Directory where to write sitemap files. By default, the current directory.
115
116
=item B<--verbose|-v>
117
118
Enable script verbose mode: each minute, the number of biblio record links
119
written is displayed.
120
121
=item B<--help|-h>
122
123
Print this help page.
124
125
=back
126
127
=cut
128
(-)a/t/Sitemapper.t (-1 / +10 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
#
3
use Modern::Perl;
4
5
use Test::More tests => 2;
6
7
BEGIN {
8
    use_ok('Koha::Sitemapper');
9
    use_ok('Koha::Sitemapper::Writer');
10
}

Return to bug 11190