View | Details | Raw Unified | Return to bug 7818
Collapse All | Expand All

(-)a/Koha/Indexer/Utils.pm (+222 lines)
Line 0 Link Here
1
package Koha::Indexer::Utils;
2
3
# Copyright (c) 2012 Equinox Software, Inc.
4
# This file is part of Koha.
5
#
6
# Koha is free software; you can redistribute it and/or modify it under the
7
# terms of the GNU General Public License as published by the Free Software
8
# Foundation; either version 2 of the License, or (at your option) any later
9
# version.
10
#
11
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
12
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License along with
16
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
17
# Suite 330, Boston, MA  02111-1307 USA
18
19
use strict;
20
use warnings;
21
use 5.010;
22
23
use XML::LibXML;
24
25
=head1 Koha::Indexer::Utils
26
27
Koha::Indexer::Utils - utility functions for managing search indexes
28
29
=head1 DESCRIPTION
30
31
This modules contains utility functions for managing various aspects
32
of Koha's bibliographic and authority search indexes.
33
34
=head1 FUNCTIONS
35
36
=cut
37
38
=head2 zebra_record_abs_to_dom
39
40
$dom_config = Koha::Indexer::Utils::zebra_record_abs_to_dom($record_abs_config, $marcflavour);
41
42
Given a string containing the contents of a records.abs configuration file as
43
used by Zebra's GRS-1 filter, emit an equivalent DOM configuration.
44
45
=cut
46
47
our $idxNS = 'http://www.koha-community.org/schemas/index-defs';
48
49
sub zebra_record_abs_to_dom {
50
    my $grs1_cfg = shift;
51
    my $marcflavour = shift;
52
   
53
    chomp $grs1_cfg;
54
    my @grs1_cfg_lines = split /\n/, $grs1_cfg, -1;
55
    my $grs1_defs = [];
56
57
    # generate an arrayref of structures representing
58
    # each records.abs line
59
    for (my $i = 0; $i <= $#grs1_cfg_lines; $i++) {
60
        my $line = $grs1_cfg_lines[$i];
61
        next if _can_ignore_grs1_cfg_line($line);
62
        my $grs1_def = _parse_grs1_cfg_line($line);
63
        $grs1_def->{orig_def} = $line;
64
        $grs1_def->{lineno} = $i + 1;
65
        push @$grs1_defs, $grs1_def;
66
    }
67
68
    # map the index definitions to a DOM tree representing
69
    # the index definitions -- if you squint hard, you
70
    # can see the beginnings of a more general definition language
71
    # for Koha index definitions
72
    my $dom_cfg = XML::LibXML::Document->new('1.0', 'utf-8');
73
    my $root = $dom_cfg->createElement('index_defs');
74
    $root->setNamespace($idxNS, 'kohaidx');
75
    foreach my $grs1_def (@$grs1_defs) {
76
        _append_grs1_def_to_dom_cfg($dom_cfg, $root, $grs1_def, $marcflavour);
77
    }
78
79
    # and emit the result as a string
80
    $dom_cfg->setDocumentElement($root);
81
    return $dom_cfg->toString(1);
82
}
83
84
#
85
# bunch of utility functions for zebra_record_abs_to_dom
86
#
87
sub _can_ignore_grs1_cfg_line {
88
    my $line = shift;
89
    return 1 if $line =~ /^\s*$/ or
90
                $line =~ /^#/ or
91
                $line =~ /^(encoding|name|attset|esetname|marc|systag|xpath)/ or
92
                $line =~ /^all/; # DOM filter automatically indexes all tokens, so
93
                                 # no need to deal with 'all any' lines in record.abs
94
    return 0;
95
}
96
97
sub _parse_grs1_cfg_line {
98
    my $line = shift;
99
    my $grs1_def;
100
   
101
    if ($line =~ /^melm\s+(.*)/ || $line =~ m!^xelm /record/(.*)!) {
102
        $grs1_def = _parse_xelm_melm($1);
103
    }
104
    return $grs1_def;
105
}
106
107
sub _parse_xelm_melm {
108
    my $line = shift;
109
   
110
    my ($field, $index_defs) = split /\s+/, $line, 2; 
111
112
    # munge fixed field range indicators
113
    $index_defs =~ s/range\(data,(\d+),(\d+)\)/$1:$2/g;
114
115
    my ($tag, $subfield) = split /\$/, $field, 2;
116
    return {
117
        tag         => $tag,
118
        subfield    => $subfield,
119
        index_defs  => [ map { _parse_grs1_index_def($_) } split /,/, $index_defs ],
120
    };   
121
}
122
123
sub _parse_grs1_index_def {
124
    my $index_def = shift;
125
126
    my @parts = split /:/, $index_def, -1;
127
    my $parsed_def = {};
128
    $parsed_def->{name}       = shift @parts;
129
    $parsed_def->{index_type} = shift @parts;
130
    $parsed_def->{offset}     = shift @parts;
131
    $parsed_def->{length}     = shift @parts;
132
    # if the original index definition didn't specify an index
133
    # type, set it 'w' -- the DOM filter needs the index type
134
    # to be specified explicitly
135
    $parsed_def->{index_type} = 'w' unless defined $parsed_def->{index_type};
136
    return $parsed_def;
137
}
138
139
sub _append_grs1_def_to_dom_cfg {
140
    my $dom_cfg = shift;
141
    my $root = shift;
142
    my $grs1_def = shift;
143
    my $marcflavour = shift;
144
145
    my $comment = $dom_cfg->createComment('record.abs line ' . 
146
                                          $grs1_def->{lineno} . ': ' .
147
                                          $grs1_def->{orig_def});
148
    $root->appendChild($comment);
149
150
    if (defined $grs1_def->{tag} && defined $grs1_def->{subfield}) {
151
        my $dom_def = $dom_cfg->createElementNS($idxNS, 'index_subfields');
152
        $dom_def->setAttribute('tag', $grs1_def->{tag});
153
        $dom_def->setAttribute('subfields', $grs1_def->{subfield});
154
        _append_target_indexes($dom_cfg, $dom_def, $grs1_def);
155
        $root->appendChild($dom_def);
156
    } elsif (defined $grs1_def->{tag} and $grs1_def->{tag} eq 'leader') {
157
        # we're the leader
158
        _append_grs1_defs_for_leader($dom_cfg, $root, $grs1_def);
159
    } elsif (defined $grs1_def->{tag} and $grs1_def->{tag} < 10) {
160
        # we're a control field
161
        _append_grs1_defs_for_control_field($dom_cfg, $root, $grs1_def);
162
    } elsif (defined $grs1_def->{tag}) {
163
        # we're indexing an entire variable data field
164
        my $dom_def = $dom_cfg->createElementNS($idxNS, 'index_data_field');
165
        $dom_def->setAttribute('tag', $grs1_def->{tag});
166
        _append_target_indexes($dom_cfg, $dom_def, $grs1_def);
167
        $root->appendChild($dom_def);
168
    }
169
} 
170
171
sub _append_target_indexes { 
172
    my $dom_cfg = shift;
173
    my $dom_def = shift;
174
    my $grs1_def = shift;
175
176
    foreach my $index_def (@{ $grs1_def->{index_defs} }) {
177
        _append_one_target_index($dom_cfg, $dom_def, $index_def);
178
    }
179
}
180
181
sub _append_one_target_index {
182
    my $dom_cfg = shift;
183
    my $dom_def = shift;
184
    my $index_def = shift;
185
    my $tgt_idx = $dom_cfg->createElementNS($idxNS, 'target_index');
186
    my $index_name = "$index_def->{name}:$index_def->{index_type}"; 
187
    $tgt_idx->appendText($index_name);
188
    $dom_def->appendChild($tgt_idx);
189
}
190
191
sub _append_grs1_defs_for_leader {
192
    my $dom_cfg = shift;
193
    my $root = shift;
194
    my $grs1_def = shift;
195
    foreach my $index_def (@{ $grs1_def->{index_defs} }) {
196
        my $dom_def = $dom_cfg->createElementNS($idxNS, 'index_leader');
197
        if (defined $index_def->{offset} && defined $index_def->{length}) {
198
            $dom_def->setAttribute('offset', $index_def->{offset});
199
            $dom_def->setAttribute('length', $index_def->{length});
200
        }
201
        _append_one_target_index($dom_cfg, $dom_def, $index_def);
202
        $root->appendChild($dom_def);
203
    }
204
}
205
206
sub _append_grs1_defs_for_control_field {
207
    my $dom_cfg = shift;
208
    my $root = shift;
209
    my $grs1_def = shift;
210
    foreach my $index_def (@{ $grs1_def->{index_defs} }) {
211
        my $dom_def = $dom_cfg->createElementNS($idxNS, 'index_control_field');
212
        $dom_def->setAttribute('tag', $grs1_def->{tag});
213
        if (defined $index_def->{offset} && defined $index_def->{length}) {
214
            $dom_def->setAttribute('offset', $index_def->{offset});
215
            $dom_def->setAttribute('length', $index_def->{length});
216
        }
217
        _append_one_target_index($dom_cfg, $dom_def, $index_def);
218
        $root->appendChild($dom_def);
219
    }
220
}
221
222
1;
(-)a/misc/maintenance/make_zebra_dom_cfg_from_record_abs (-1 / +72 lines)
Line 0 Link Here
0
- 
1
#!/usr/bin/perl
2
3
# Copyright (c) 2012 Equinox Software, Inc.
4
# This file is part of Koha.
5
#
6
# Koha is free software; you can redistribute it and/or modify it under the
7
# terms of the GNU General Public License as published by the Free Software
8
# Foundation; either version 2 of the License, or (at your option) any later
9
# version.
10
#
11
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
12
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
13
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
14
#
15
# You should have received a copy of the GNU General Public License along with
16
# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place,
17
# Suite 330, Boston, MA  02111-1307 USA
18
19
use strict;
20
use warnings;
21
use 5.010;
22
23
use Koha::Indexer::Utils;
24
25
use Getopt::Long;
26
27
my $input_file;
28
my $output_file;
29
my $want_help;
30
my $result = GetOptions(
31
    'input:s'      => \$input_file,
32
    'output:s'     => \$output_file,
33
    'help|h'       => \$want_help,
34
);
35
36
if ( not $result or $want_help or not defined $input_file or not defined $output_file ) {
37
    print_usage();
38
    exit 0;
39
}
40
41
open my $infh,  '<', $input_file or die "$0: cannot open input file $input_file: $!\n";
42
open my $outfh, '>', $output_file or die "$0: cannot open output file $output_file: $!\n";
43
44
my $grs1_cfg = join('', <$infh>);
45
close $infh;
46
my $dom_cfg = Koha::Indexer::Utils::zebra_record_abs_to_dom($grs1_cfg);
47
print $outfh $dom_cfg;
48
close $outfh;
49
50
sub print_usage {
51
    print <<_USAGE_;
52
$0: generate a DOM filter Zebra index config from a GRS-1 config
53
54
Given a Zebra record.abs file containing a set of index definitions for
55
Zebra's GRS-1 filter, write an equivalent DOM filter configuration.
56
57
To generate the XSLT that is to be used by Zebra, run something like
58
the following on the output of this utility:
59
60
xsltproc ZEBRA_CFG_DIR/xsl/koha-indexdefs-to-zebra.xsl \\
61
  biblio-koha-indexdefs.xml \\
62
  > ZEBRA_CFG_DIR/marc_defs/marc21/biblios/biblio-zebra-indexdefs.xsl
63
64
The above example assumes that the output of the program was named 
65
biblio-koha-indexdefs.xsl.
66
67
Parameters:
68
    --input                 input file name
69
    --output                output file name
70
    --help or -h            show this message
71
_USAGE_
72
}

Return to bug 7818