Lines 35-41
use Try::Tiny;
Link Here
|
35 |
use YAML::Syck; |
35 |
use YAML::Syck; |
36 |
|
36 |
|
37 |
use List::Util qw( sum0 reduce ); |
37 |
use List::Util qw( sum0 reduce ); |
38 |
use Search::Elasticsearch; |
|
|
39 |
use MARC::File::XML; |
38 |
use MARC::File::XML; |
40 |
use MIME::Base64; |
39 |
use MIME::Base64; |
41 |
use Encode qw(encode); |
40 |
use Encode qw(encode); |
Lines 69-75
sub new {
Link Here
|
69 |
my $class = shift @_; |
68 |
my $class = shift @_; |
70 |
my $self = $class->SUPER::new(@_); |
69 |
my $self = $class->SUPER::new(@_); |
71 |
# Check for a valid index |
70 |
# Check for a valid index |
72 |
croak('No index name provided') unless $self->index; |
71 |
Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index; |
73 |
return $self; |
72 |
return $self; |
74 |
} |
73 |
} |
75 |
|
74 |
|
Lines 312-328
sub sort_fields {
Link Here
|
312 |
|
311 |
|
313 |
=head2 _process_mappings($mappings, $data, $record_document) |
312 |
=head2 _process_mappings($mappings, $data, $record_document) |
314 |
|
313 |
|
315 |
Process all C<$mappings> targets operating on a specific MARC field C<$data> applied to C<$record_document> |
314 |
$self->_process_mappings($mappings, $marc_field_data, $record_document) |
316 |
Since we group all mappings by MARC field targets C<$mappings> will contain all targets for C<$data> |
315 |
|
317 |
and thus we need to fetch the MARC field only once. |
316 |
Process all C<$mappings> targets operating on a specific MARC field C<$data>. |
|
|
317 |
Since we group all mappings by MARC field targets C<$mappings> will contain |
318 |
all targets for C<$data> and thus we need to fetch the MARC field only once. |
319 |
C<$mappings> will be applied to C<$record_document> and new field values added. |
320 |
The method has no return value. |
318 |
|
321 |
|
319 |
=over 4 |
322 |
=over 4 |
320 |
|
323 |
|
321 |
=item C<$mappings> |
324 |
=item C<$mappings> |
322 |
|
325 |
|
323 |
Arrayref of mappings containing arrayrefs on the format [C<$taget>, C<$options>] where |
326 |
Arrayref of mappings containing arrayrefs in the format |
324 |
C<$target> is the name of the target field and C<$options> is a hashref containing processing |
327 |
[C<$taget>, C<$options>] where C<$target> is the name of the target field and |
325 |
directives for this particular mapping. |
328 |
C<$options> is a hashref containing processing directives for this particular |
|
|
329 |
mapping. |
326 |
|
330 |
|
327 |
=item C<$data> |
331 |
=item C<$data> |
328 |
|
332 |
|
Lines 330-336
The source data from a MARC record field.
Link Here
|
330 |
|
334 |
|
331 |
=item C<$record_document> |
335 |
=item C<$record_document> |
332 |
|
336 |
|
333 |
Hashref representing the Elasticsearch document on which mappings should be applied. |
337 |
Hashref representing the Elasticsearch document on which mappings should be |
|
|
338 |
applied. |
334 |
|
339 |
|
335 |
=back |
340 |
=back |
336 |
|
341 |
|
Lines 465-471
sub marc_records_to_documents {
Link Here
|
465 |
# Suppress warnings if record length exceeded |
470 |
# Suppress warnings if record length exceeded |
466 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
471 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
467 |
foreach my $warning (@warnings) { |
472 |
foreach my $warning (@warnings) { |
468 |
carp($warning); |
473 |
carp $warning; |
469 |
} |
474 |
} |
470 |
} |
475 |
} |
471 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
476 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
Lines 482-496
sub marc_records_to_documents {
Link Here
|
482 |
|
487 |
|
483 |
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
488 |
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
484 |
|
489 |
|
485 |
Get mappings, an internal data structure later used by L<_process_mappings($mappings, $data, $record_document)> |
490 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
486 |
to process MARC target data, for a MARC mapping. |
|
|
487 |
|
491 |
|
488 |
The returned C<$mappings> is to to be confused with mappings provided by C<_foreach_mapping>, rather this |
492 |
Get mappings, an internal data structure later used by |
489 |
sub accepts properties from a mapping as provided by C<_foreach_mapping> and expands it to this internal |
493 |
L<_process_mappings($mappings, $data, $record_document)> to process MARC target |
490 |
data stucture. In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> is then |
494 |
data for a MARC mapping. |
491 |
applied to each MARC target (leader, control field data, subfield or joined subfields) and |
495 |
|
492 |
integrated into the mapping rules data structure used in C<marc_records_to_documents> to |
496 |
The returned C<$mappings> is not to to be confused with mappings provided by |
493 |
transform MARC records into Elasticsearch documents. |
497 |
C<_foreach_mapping>, rather this sub accepts properties from a mapping as |
|
|
498 |
provided by C<_foreach_mapping> and expands it to this internal data stucture. |
499 |
In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> |
500 |
is then applied to each MARC target (leader, control field data, subfield or |
501 |
joined subfields) and integrated into the mapping rules data structure used in |
502 |
C<marc_records_to_documents> to transform MARC records into Elasticsearch |
503 |
documents. |
494 |
|
504 |
|
495 |
=over 4 |
505 |
=over 4 |
496 |
|
506 |
|
Lines 516-529
Elasticsearch document target field type.
Link Here
|
516 |
|
526 |
|
517 |
=item C<$range> |
527 |
=item C<$range> |
518 |
|
528 |
|
519 |
An optinal range as a string on the format "<START>-<END>" or "<START>", |
529 |
An optional range as a string in the format "<START>-<END>" or "<START>", |
520 |
where "<START>" and "<END>" are integers specifying a range that will be used |
530 |
where "<START>" and "<END>" are integers specifying a range that will be used |
521 |
for extracting a substing from MARC data as Elasticsearch field target value. |
531 |
for extracting a substring from MARC data as Elasticsearch field target value. |
522 |
|
532 |
|
523 |
The first character position is "1", and the range is inclusive, |
533 |
The first character position is "1", and the range is inclusive, |
524 |
so "1-3" means the first three characters of MARC data. |
534 |
so "1-3" means the first three characters of MARC data. |
525 |
|
535 |
|
526 |
If only "<START>" is provided only one character as position "<START>" will |
536 |
If only "<START>" is provided only one character at position "<START>" will |
527 |
be extracted. |
537 |
be extracted. |
528 |
|
538 |
|
529 |
=back |
539 |
=back |
Lines 593-599
rules keyed by MARC field tags holding all the mapping rules for that particular
Link Here
|
593 |
|
603 |
|
594 |
We can then iterate through all MARC fields for each record and apply all relevant |
604 |
We can then iterate through all MARC fields for each record and apply all relevant |
595 |
rules once per fields instead of retreiving fields multiple times for each mapping rule |
605 |
rules once per fields instead of retreiving fields multiple times for each mapping rule |
596 |
wich is terribly slow. |
606 |
which is terribly slow. |
597 |
|
607 |
|
598 |
=cut |
608 |
=cut |
599 |
|
609 |
|
Lines 604-613
wich is terribly slow.
Link Here
|
604 |
|
614 |
|
605 |
sub _get_marc_mapping_rules { |
615 |
sub _get_marc_mapping_rules { |
606 |
my ($self) = @_; |
616 |
my ($self) = @_; |
607 |
|
|
|
608 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
617 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
609 |
my @rules; |
|
|
610 |
|
611 |
my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
618 |
my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
612 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
619 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
613 |
my $rules = { |
620 |
my $rules = { |
Lines 625-631
sub _get_marc_mapping_rules {
Link Here
|
625 |
if ($type eq 'sum') { |
632 |
if ($type eq 'sum') { |
626 |
push @{$rules->{sum}}, $name; |
633 |
push @{$rules->{sum}}, $name; |
627 |
} |
634 |
} |
628 |
elsif($type eq 'boolean') { |
635 |
elsif ($type eq 'boolean') { |
629 |
# boolean gets special handling, if value doesn't exist for a field, |
636 |
# boolean gets special handling, if value doesn't exist for a field, |
630 |
# it is set to false |
637 |
# it is set to false |
631 |
$rules->{defaults}->{$name} = 'false'; |
638 |
$rules->{defaults}->{$name} = 'false'; |
Lines 644-650
sub _get_marc_mapping_rules {
Link Here
|
644 |
foreach my $token (split //, $2) { |
651 |
foreach my $token (split //, $2) { |
645 |
if ($token eq "(") { |
652 |
if ($token eq "(") { |
646 |
if ($open_group) { |
653 |
if ($open_group) { |
647 |
die("Unmatched opening parenthesis for $marc_field"); |
654 |
Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw( |
|
|
655 |
"Unmatched opening parenthesis for $marc_field" |
656 |
); |
648 |
} |
657 |
} |
649 |
else { |
658 |
else { |
650 |
$open_group = 1; |
659 |
$open_group = 1; |
Lines 659-665
sub _get_marc_mapping_rules {
Link Here
|
659 |
$open_group = 0; |
668 |
$open_group = 0; |
660 |
} |
669 |
} |
661 |
else { |
670 |
else { |
662 |
die("Unmatched closing parenthesis for $marc_field"); |
671 |
Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw( |
|
|
672 |
"Unmatched closing parenthesis for $marc_field" |
673 |
); |
663 |
} |
674 |
} |
664 |
} |
675 |
} |
665 |
elsif ($open_group) { |
676 |
elsif ($open_group) { |
Lines 699-705
sub _get_marc_mapping_rules {
Link Here
|
699 |
push @{$rules->{leader}}, @mappings; |
710 |
push @{$rules->{leader}}, @mappings; |
700 |
} |
711 |
} |
701 |
else { |
712 |
else { |
702 |
die("Invalid MARC field: $marc_field"); |
713 |
Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw( |
|
|
714 |
"Invalid MARC field expression: $marc_field" |
715 |
); |
703 |
} |
716 |
} |
704 |
}); |
717 |
}); |
705 |
return $rules; |
718 |
return $rules; |