|
Lines 35-41
use Try::Tiny;
Link Here
|
| 35 |
use YAML::Syck; |
35 |
use YAML::Syck; |
| 36 |
|
36 |
|
| 37 |
use List::Util qw( sum0 reduce ); |
37 |
use List::Util qw( sum0 reduce ); |
| 38 |
use Search::Elasticsearch; |
|
|
| 39 |
use MARC::File::XML; |
38 |
use MARC::File::XML; |
| 40 |
use MIME::Base64; |
39 |
use MIME::Base64; |
| 41 |
use Encode qw(encode); |
40 |
use Encode qw(encode); |
|
Lines 69-75
sub new {
Link Here
|
| 69 |
my $class = shift @_; |
68 |
my $class = shift @_; |
| 70 |
my $self = $class->SUPER::new(@_); |
69 |
my $self = $class->SUPER::new(@_); |
| 71 |
# Check for a valid index |
70 |
# Check for a valid index |
| 72 |
croak('No index name provided') unless $self->index; |
71 |
Koha::Exceptions::MissingParameter->throw('No index name provided') unless $self->index; |
| 73 |
return $self; |
72 |
return $self; |
| 74 |
} |
73 |
} |
| 75 |
|
74 |
|
|
Lines 312-328
sub sort_fields {
Link Here
|
| 312 |
|
311 |
|
| 313 |
=head2 _process_mappings($mappings, $data, $record_document) |
312 |
=head2 _process_mappings($mappings, $data, $record_document) |
| 314 |
|
313 |
|
| 315 |
Process all C<$mappings> targets operating on a specific MARC field C<$data> applied to C<$record_document> |
314 |
$self->_process_mappings($mappings, $marc_field_data, $record_document) |
| 316 |
Since we group all mappings by MARC field targets C<$mappings> will contain all targets for C<$data> |
315 |
|
| 317 |
and thus we need to fetch the MARC field only once. |
316 |
Process all C<$mappings> targets operating on a specific MARC field C<$data>. |
|
|
317 |
Since we group all mappings by MARC field targets C<$mappings> will contain |
| 318 |
all targets for C<$data> and thus we need to fetch the MARC field only once. |
| 319 |
C<$mappings> will be applied to C<$record_document> and new field values added. |
| 320 |
The method has no return value. |
| 318 |
|
321 |
|
| 319 |
=over 4 |
322 |
=over 4 |
| 320 |
|
323 |
|
| 321 |
=item C<$mappings> |
324 |
=item C<$mappings> |
| 322 |
|
325 |
|
| 323 |
Arrayref of mappings containing arrayrefs on the format [C<$taget>, C<$options>] where |
326 |
Arrayref of mappings containing arrayrefs in the format |
| 324 |
C<$target> is the name of the target field and C<$options> is a hashref containing processing |
327 |
[C<$taget>, C<$options>] where C<$target> is the name of the target field and |
| 325 |
directives for this particular mapping. |
328 |
C<$options> is a hashref containing processing directives for this particular |
|
|
329 |
mapping. |
| 326 |
|
330 |
|
| 327 |
=item C<$data> |
331 |
=item C<$data> |
| 328 |
|
332 |
|
|
Lines 330-336
The source data from a MARC record field.
Link Here
|
| 330 |
|
334 |
|
| 331 |
=item C<$record_document> |
335 |
=item C<$record_document> |
| 332 |
|
336 |
|
| 333 |
Hashref representing the Elasticsearch document on which mappings should be applied. |
337 |
Hashref representing the Elasticsearch document on which mappings should be |
|
|
338 |
applied. |
| 334 |
|
339 |
|
| 335 |
=back |
340 |
=back |
| 336 |
|
341 |
|
|
Lines 465-471
sub marc_records_to_documents {
Link Here
|
| 465 |
# Suppress warnings if record length exceeded |
470 |
# Suppress warnings if record length exceeded |
| 466 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
471 |
unless (substr($record->leader(), 0, 5) eq '99999') { |
| 467 |
foreach my $warning (@warnings) { |
472 |
foreach my $warning (@warnings) { |
| 468 |
carp($warning); |
473 |
carp $warning; |
| 469 |
} |
474 |
} |
| 470 |
} |
475 |
} |
| 471 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
476 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
|
Lines 482-496
sub marc_records_to_documents {
Link Here
|
| 482 |
|
487 |
|
| 483 |
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
488 |
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
| 484 |
|
489 |
|
| 485 |
Get mappings, an internal data structure later used by L<_process_mappings($mappings, $data, $record_document)> |
490 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
| 486 |
to process MARC target data, for a MARC mapping. |
|
|
| 487 |
|
491 |
|
| 488 |
The returned C<$mappings> is to to be confused with mappings provided by C<_foreach_mapping>, rather this |
492 |
Get mappings, an internal data structure later used by |
| 489 |
sub accepts properties from a mapping as provided by C<_foreach_mapping> and expands it to this internal |
493 |
L<_process_mappings($mappings, $data, $record_document)> to process MARC target |
| 490 |
data stucture. In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> is then |
494 |
data for a MARC mapping. |
| 491 |
applied to each MARC target (leader, control field data, subfield or joined subfields) and |
495 |
|
| 492 |
integrated into the mapping rules data structure used in C<marc_records_to_documents> to |
496 |
The returned C<$mappings> is not to to be confused with mappings provided by |
| 493 |
transform MARC records into Elasticsearch documents. |
497 |
C<_foreach_mapping>, rather this sub accepts properties from a mapping as |
|
|
498 |
provided by C<_foreach_mapping> and expands it to this internal data stucture. |
| 499 |
In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> |
| 500 |
is then applied to each MARC target (leader, control field data, subfield or |
| 501 |
joined subfields) and integrated into the mapping rules data structure used in |
| 502 |
C<marc_records_to_documents> to transform MARC records into Elasticsearch |
| 503 |
documents. |
| 494 |
|
504 |
|
| 495 |
=over 4 |
505 |
=over 4 |
| 496 |
|
506 |
|
|
Lines 516-529
Elasticsearch document target field type.
Link Here
|
| 516 |
|
526 |
|
| 517 |
=item C<$range> |
527 |
=item C<$range> |
| 518 |
|
528 |
|
| 519 |
An optinal range as a string on the format "<START>-<END>" or "<START>", |
529 |
An optional range as a string in the format "<START>-<END>" or "<START>", |
| 520 |
where "<START>" and "<END>" are integers specifying a range that will be used |
530 |
where "<START>" and "<END>" are integers specifying a range that will be used |
| 521 |
for extracting a substing from MARC data as Elasticsearch field target value. |
531 |
for extracting a substring from MARC data as Elasticsearch field target value. |
| 522 |
|
532 |
|
| 523 |
The first character position is "1", and the range is inclusive, |
533 |
The first character position is "1", and the range is inclusive, |
| 524 |
so "1-3" means the first three characters of MARC data. |
534 |
so "1-3" means the first three characters of MARC data. |
| 525 |
|
535 |
|
| 526 |
If only "<START>" is provided only one character as position "<START>" will |
536 |
If only "<START>" is provided only one character at position "<START>" will |
| 527 |
be extracted. |
537 |
be extracted. |
| 528 |
|
538 |
|
| 529 |
=back |
539 |
=back |
|
Lines 593-599
rules keyed by MARC field tags holding all the mapping rules for that particular
Link Here
|
| 593 |
|
603 |
|
| 594 |
We can then iterate through all MARC fields for each record and apply all relevant |
604 |
We can then iterate through all MARC fields for each record and apply all relevant |
| 595 |
rules once per fields instead of retreiving fields multiple times for each mapping rule |
605 |
rules once per fields instead of retreiving fields multiple times for each mapping rule |
| 596 |
wich is terribly slow. |
606 |
which is terribly slow. |
| 597 |
|
607 |
|
| 598 |
=cut |
608 |
=cut |
| 599 |
|
609 |
|
|
Lines 604-613
wich is terribly slow.
Link Here
|
| 604 |
|
614 |
|
| 605 |
sub _get_marc_mapping_rules { |
615 |
sub _get_marc_mapping_rules { |
| 606 |
my ($self) = @_; |
616 |
my ($self) = @_; |
| 607 |
|
|
|
| 608 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
617 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 609 |
my @rules; |
|
|
| 610 |
|
| 611 |
my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
618 |
my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
| 612 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
619 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
| 613 |
my $rules = { |
620 |
my $rules = { |
|
Lines 625-631
sub _get_marc_mapping_rules {
Link Here
|
| 625 |
if ($type eq 'sum') { |
632 |
if ($type eq 'sum') { |
| 626 |
push @{$rules->{sum}}, $name; |
633 |
push @{$rules->{sum}}, $name; |
| 627 |
} |
634 |
} |
| 628 |
elsif($type eq 'boolean') { |
635 |
elsif ($type eq 'boolean') { |
| 629 |
# boolean gets special handling, if value doesn't exist for a field, |
636 |
# boolean gets special handling, if value doesn't exist for a field, |
| 630 |
# it is set to false |
637 |
# it is set to false |
| 631 |
$rules->{defaults}->{$name} = 'false'; |
638 |
$rules->{defaults}->{$name} = 'false'; |
|
Lines 644-650
sub _get_marc_mapping_rules {
Link Here
|
| 644 |
foreach my $token (split //, $2) { |
651 |
foreach my $token (split //, $2) { |
| 645 |
if ($token eq "(") { |
652 |
if ($token eq "(") { |
| 646 |
if ($open_group) { |
653 |
if ($open_group) { |
| 647 |
die("Unmatched opening parenthesis for $marc_field"); |
654 |
Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw( |
|
|
655 |
"Unmatched opening parenthesis for $marc_field" |
| 656 |
); |
| 648 |
} |
657 |
} |
| 649 |
else { |
658 |
else { |
| 650 |
$open_group = 1; |
659 |
$open_group = 1; |
|
Lines 659-665
sub _get_marc_mapping_rules {
Link Here
|
| 659 |
$open_group = 0; |
668 |
$open_group = 0; |
| 660 |
} |
669 |
} |
| 661 |
else { |
670 |
else { |
| 662 |
die("Unmatched closing parenthesis for $marc_field"); |
671 |
Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw( |
|
|
672 |
"Unmatched closing parenthesis for $marc_field" |
| 673 |
); |
| 663 |
} |
674 |
} |
| 664 |
} |
675 |
} |
| 665 |
elsif ($open_group) { |
676 |
elsif ($open_group) { |
|
Lines 699-705
sub _get_marc_mapping_rules {
Link Here
|
| 699 |
push @{$rules->{leader}}, @mappings; |
710 |
push @{$rules->{leader}}, @mappings; |
| 700 |
} |
711 |
} |
| 701 |
else { |
712 |
else { |
| 702 |
die("Invalid MARC field: $marc_field"); |
713 |
Koha::Exceptions::Elasticsearch::MARCFieldExprParseError->throw( |
|
|
714 |
"Invalid MARC field expression: $marc_field" |
| 715 |
); |
| 703 |
} |
716 |
} |
| 704 |
}); |
717 |
}); |
| 705 |
return $rules; |
718 |
return $rules; |