|
Lines 73-78
sub new {
Link Here
|
| 73 |
return $self; |
73 |
return $self; |
| 74 |
} |
74 |
} |
| 75 |
|
75 |
|
|
|
76 |
=head2 get_elasticsearch |
| 77 |
|
| 78 |
my $elasticsearch_client = $self->get_elasticsearch(); |
| 79 |
|
| 80 |
Returns a C<Search::Elasticsearch> client. The client is cached on a C<Koha::SearchEngine::ElasticSearch> |
| 81 |
instance level and will be reused if method is called multiple times. |
| 82 |
|
| 83 |
=cut |
| 84 |
|
| 76 |
sub get_elasticsearch { |
85 |
sub get_elasticsearch { |
| 77 |
my $self = shift @_; |
86 |
my $self = shift @_; |
| 78 |
unless (defined $self->{elasticsearch}) { |
87 |
unless (defined $self->{elasticsearch}) { |
|
Lines 145-152
sub get_elasticsearch_params {
Link Here
|
| 145 |
|
154 |
|
| 146 |
my $settings = $self->get_elasticsearch_settings(); |
155 |
my $settings = $self->get_elasticsearch_settings(); |
| 147 |
|
156 |
|
| 148 |
This provides the settings provided to elasticsearch when an index is created. |
157 |
This provides the settings provided to Elasticsearch when an index is created. |
| 149 |
These can do things like define tokenisation methods. |
158 |
These can do things like define tokenization methods. |
| 150 |
|
159 |
|
| 151 |
A hashref containing the settings is returned. |
160 |
A hashref containing the settings is returned. |
| 152 |
|
161 |
|
|
Lines 170-176
sub get_elasticsearch_settings {
Link Here
|
| 170 |
|
179 |
|
| 171 |
my $mappings = $self->get_elasticsearch_mappings(); |
180 |
my $mappings = $self->get_elasticsearch_mappings(); |
| 172 |
|
181 |
|
| 173 |
This provides the mappings that get passed to elasticsearch when an index is |
182 |
This provides the mappings that get passed to Elasticsearch when an index is |
| 174 |
created. |
183 |
created. |
| 175 |
|
184 |
|
| 176 |
=cut |
185 |
=cut |
|
Lines 232-238
sub get_elasticsearch_mappings {
Link Here
|
| 232 |
|
241 |
|
| 233 |
=head2 _get_elasticsearch_mapping |
242 |
=head2 _get_elasticsearch_mapping |
| 234 |
|
243 |
|
| 235 |
Get the ES mappings for the given purpose and data type |
244 |
Get the Elasticsearch mappings for the given purpose and data type. |
| 236 |
|
245 |
|
| 237 |
$mapping = _get_elasticsearch_mapping('search', 'text'); |
246 |
$mapping = _get_elasticsearch_mapping('search', 'text'); |
| 238 |
|
247 |
|
|
Lines 301-351
sub sort_fields {
Link Here
|
| 301 |
return $self->_sort_fields_accessor(); |
310 |
return $self->_sort_fields_accessor(); |
| 302 |
} |
311 |
} |
| 303 |
|
312 |
|
|
|
313 |
=head2 _process_mappings($mappings, $data, $record_document) |
| 314 |
|
| 315 |
Process all C<$mappings> targets operating on a specific MARC field C<$data> applied to C<$record_document> |
| 316 |
Since we group all mappings by MARC field targets C<$mappings> will contain all targets for C<$data> |
| 317 |
and thus we need to fetch the MARC field only once. |
| 318 |
|
| 319 |
=over 4 |
| 320 |
|
| 321 |
=item C<$mappings> |
| 322 |
|
| 323 |
Arrayref of mappings containing arrayrefs on the format [C<$taget>, C<$options>] where |
| 324 |
C<$target> is the name of the target field and C<$options> is a hashref containing processing |
| 325 |
directives for this particular mapping. |
| 326 |
|
| 327 |
=item C<$data> |
| 328 |
|
| 329 |
The source data from a MARC record field. |
| 330 |
|
| 331 |
=item C<$record_document> |
| 332 |
|
| 333 |
Hashref representing the Elasticsearch document on which mappings should be applied. |
| 334 |
|
| 335 |
=back |
| 336 |
|
| 337 |
=cut |
| 338 |
|
| 339 |
sub _process_mappings { |
| 340 |
my ($_self, $mappings, $data, $record_document) = @_; |
| 341 |
foreach my $mapping (@{$mappings}) { |
| 342 |
my ($target, $options) = @{$mapping}; |
| 343 |
# Copy (scalar) data since can have multiple targets |
| 344 |
# with differing options for (possibly) mutating data |
| 345 |
# so need a different copy for each |
| 346 |
my $_data = $data; |
| 347 |
$record_document->{$target} //= []; |
| 348 |
if (defined $options->{substr}) { |
| 349 |
my ($start, $length) = @{$options->{substr}}; |
| 350 |
$_data = length($data) > $start ? substr $data, $start, $length : ''; |
| 351 |
} |
| 352 |
if (defined $options->{value_callbacks}) { |
| 353 |
$_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}}); |
| 354 |
} |
| 355 |
if (defined $options->{property}) { |
| 356 |
$_data = { |
| 357 |
$options->{property} => $_data |
| 358 |
} |
| 359 |
} |
| 360 |
push @{$record_document->{$target}}, $_data; |
| 361 |
} |
| 362 |
} |
| 363 |
|
| 364 |
=head2 marc_records_to_documents($marc_records) |
| 365 |
|
| 366 |
my @record_documents = $self->marc_records_to_documents($marc_records); |
| 367 |
|
| 368 |
Using mappings stored in database convert C<$marc_records> to Elasticsearch documents. |
| 369 |
|
| 370 |
Returns array of hash references, representing Elasticsearch documents, |
| 371 |
acceptable as body payload in C<Search::Elasticsearch> requests. |
| 372 |
|
| 373 |
=over 4 |
| 374 |
|
| 375 |
=item C<$marc_documents> |
| 376 |
|
| 377 |
Reference to array of C<MARC::Record> objects to be converted to Elasticsearch documents. |
| 378 |
|
| 379 |
=back |
| 380 |
|
| 381 |
=cut |
| 382 |
|
| 304 |
sub marc_records_to_documents { |
383 |
sub marc_records_to_documents { |
| 305 |
my ($self, $records) = @_; |
384 |
my ($self, $records) = @_; |
| 306 |
my $rules = $self->get_marc_mapping_rules(); |
385 |
my $rules = $self->_get_marc_mapping_rules(); |
| 307 |
my $control_fields_rules = $rules->{control_fields}; |
386 |
my $control_fields_rules = $rules->{control_fields}; |
| 308 |
my $data_fields_rules = $rules->{data_fields}; |
387 |
my $data_fields_rules = $rules->{data_fields}; |
| 309 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
388 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 310 |
my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat'); |
|
|
| 311 |
|
389 |
|
| 312 |
my @record_documents; |
390 |
my @record_documents; |
| 313 |
|
391 |
|
| 314 |
sub _process_mappings { |
|
|
| 315 |
my ($mappings, $data, $record_document) = @_; |
| 316 |
foreach my $mapping (@{$mappings}) { |
| 317 |
my ($target, $options) = @{$mapping}; |
| 318 |
# Copy (scalar) data since can have multiple targets |
| 319 |
# with differing options for (possibly) mutating data |
| 320 |
# so need a different copy for each |
| 321 |
my $_data = $data; |
| 322 |
$record_document->{$target} //= []; |
| 323 |
if (defined $options->{substr}) { |
| 324 |
my ($start, $length) = @{$options->{substr}}; |
| 325 |
$_data = length($data) > $start ? substr $data, $start, $length : ''; |
| 326 |
} |
| 327 |
if (defined $options->{value_callbacks}) { |
| 328 |
$_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}}); |
| 329 |
} |
| 330 |
if (defined $options->{property}) { |
| 331 |
$_data = { |
| 332 |
$options->{property} => $_data |
| 333 |
} |
| 334 |
} |
| 335 |
push @{$record_document->{$target}}, $_data; |
| 336 |
} |
| 337 |
} |
| 338 |
foreach my $record (@{$records}) { |
392 |
foreach my $record (@{$records}) { |
| 339 |
my $record_document = {}; |
393 |
my $record_document = {}; |
| 340 |
my $mappings = $rules->{leader}; |
394 |
my $mappings = $rules->{leader}; |
| 341 |
if ($mappings) { |
395 |
if ($mappings) { |
| 342 |
_process_mappings($mappings, $record->leader(), $record_document); |
396 |
$self->_process_mappings($mappings, $record->leader(), $record_document); |
| 343 |
} |
397 |
} |
| 344 |
foreach my $field ($record->fields()) { |
398 |
foreach my $field ($record->fields()) { |
| 345 |
if($field->is_control_field()) { |
399 |
if($field->is_control_field()) { |
| 346 |
my $mappings = $control_fields_rules->{$field->tag()}; |
400 |
my $mappings = $control_fields_rules->{$field->tag()}; |
| 347 |
if ($mappings) { |
401 |
if ($mappings) { |
| 348 |
_process_mappings($mappings, $field->data(), $record_document); |
402 |
$self->_process_mappings($mappings, $field->data(), $record_document); |
| 349 |
} |
403 |
} |
| 350 |
} |
404 |
} |
| 351 |
else { |
405 |
else { |
|
Lines 361-367
sub marc_records_to_documents {
Link Here
|
| 361 |
$mappings = [@{$mappings}, @{$wildcard_mappings}]; |
415 |
$mappings = [@{$mappings}, @{$wildcard_mappings}]; |
| 362 |
} |
416 |
} |
| 363 |
if (@{$mappings}) { |
417 |
if (@{$mappings}) { |
| 364 |
_process_mappings($mappings, $data, $record_document); |
418 |
$self->_process_mappings($mappings, $data, $record_document); |
| 365 |
} |
419 |
} |
| 366 |
} |
420 |
} |
| 367 |
|
421 |
|
|
Lines 377-383
sub marc_records_to_documents {
Link Here
|
| 377 |
) |
431 |
) |
| 378 |
); |
432 |
); |
| 379 |
if ($data) { |
433 |
if ($data) { |
| 380 |
_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document); |
434 |
$self->_process_mappings($subfields_join_mappings->{$subfields_group}, $data, $record_document); |
| 381 |
} |
435 |
} |
| 382 |
} |
436 |
} |
| 383 |
} |
437 |
} |
|
Lines 426-488
sub marc_records_to_documents {
Link Here
|
| 426 |
return \@record_documents; |
480 |
return \@record_documents; |
| 427 |
} |
481 |
} |
| 428 |
|
482 |
|
| 429 |
# Provides the rules for marc to Elasticsearch JSON document conversion. |
483 |
=head2 _field_mappings($facet, $suggestible, $sort, $target_name, $target_type, $range) |
| 430 |
sub get_marc_mapping_rules { |
|
|
| 431 |
my ($self) = @_; |
| 432 |
|
484 |
|
| 433 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
485 |
Get mappings, an internal data structure later used by L<_process_mappings($mappings, $data, $record_document)> |
| 434 |
my @rules; |
486 |
to process MARC target data, for a MARC mapping. |
|
|
487 |
|
| 488 |
The returned C<$mappings> is to to be confused with mappings provided by C<_foreach_mapping>, rather this |
| 489 |
sub accepts properties from a mapping as provided by C<_foreach_mapping> and expands it to this internal |
| 490 |
data stucture. In the caller context (C<_get_marc_mapping_rules>) the returned C<@mappings> is then |
| 491 |
applied to each MARC target (leader, control field data, subfield or joined subfields) and |
| 492 |
integrated into the mapping rules data structure used in C<marc_records_to_documents> to |
| 493 |
transform MARC records into Elasticsearch documents. |
| 494 |
|
| 495 |
=over 4 |
| 435 |
|
496 |
|
| 436 |
sub _field_mappings { |
497 |
=item C<$facet> |
| 437 |
my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_; |
498 |
|
| 438 |
my %mapping_defaults = (); |
499 |
Boolean indicating whether to create a facet field for this mapping. |
| 439 |
my @mappings; |
500 |
|
| 440 |
|
501 |
=item C<$suggestible> |
| 441 |
my $substr_args = undef; |
502 |
|
| 442 |
if ($range) { |
503 |
Boolean indicating whether to create a suggestion field for this mapping. |
| 443 |
# TODO: use value_callback instead? |
504 |
|
| 444 |
my ($start, $end) = map(int, split /-/, $range, 2); |
505 |
=item C<$sort> |
| 445 |
$substr_args = [$start]; |
506 |
|
| 446 |
push @{$substr_args}, (defined $end ? $end - $start + 1 : 1); |
507 |
Boolean indicating whether to create a sort field for this mapping. |
|
|
508 |
|
| 509 |
=item C<$target_name> |
| 510 |
|
| 511 |
Elasticsearch document target field name. |
| 512 |
|
| 513 |
=item C<$target_type> |
| 514 |
|
| 515 |
Elasticsearch document target field type. |
| 516 |
|
| 517 |
=item C<$range> |
| 518 |
|
| 519 |
An optinal range as a string on the format "<START>-<END>" or "<START>", |
| 520 |
where "<START>" and "<END>" are integers specifying a range that will be used |
| 521 |
for extracting a substing from MARC data as Elasticsearch field target value. |
| 522 |
|
| 523 |
The first character position is "1", and the range is inclusive, |
| 524 |
so "1-3" means the first three characters of MARC data. |
| 525 |
|
| 526 |
If only "<START>" is provided only one character as position "<START>" will |
| 527 |
be extracted. |
| 528 |
|
| 529 |
=back |
| 530 |
|
| 531 |
=cut |
| 532 |
|
| 533 |
sub _field_mappings { |
| 534 |
my ($_self, $facet, $suggestible, $sort, $target_name, $target_type, $range) = @_; |
| 535 |
my %mapping_defaults = (); |
| 536 |
my @mappings; |
| 537 |
|
| 538 |
my $substr_args = undef; |
| 539 |
if ($range) { |
| 540 |
# TODO: use value_callback instead? |
| 541 |
my ($start, $end) = map(int, split /-/, $range, 2); |
| 542 |
$substr_args = [$start]; |
| 543 |
push @{$substr_args}, (defined $end ? $end - $start + 1 : 1); |
| 544 |
} |
| 545 |
my $default_options = {}; |
| 546 |
if ($substr_args) { |
| 547 |
$default_options->{substr} = $substr_args; |
| 548 |
} |
| 549 |
|
| 550 |
# TODO: Should probably have per type value callback/hook |
| 551 |
# but hard code for now |
| 552 |
if ($target_type eq 'boolean') { |
| 553 |
$default_options->{value_callbacks} //= []; |
| 554 |
push @{$default_options->{value_callbacks}}, sub { |
| 555 |
my ($value) = @_; |
| 556 |
# Trim whitespace at both ends |
| 557 |
$value =~ s/^\s+|\s+$//g; |
| 558 |
return $value ? 'true' : 'false'; |
| 559 |
}; |
| 560 |
} |
| 561 |
|
| 562 |
my $mapping = [$target_name, $default_options]; |
| 563 |
push @mappings, $mapping; |
| 564 |
|
| 565 |
my @suffixes = (); |
| 566 |
push @suffixes, 'facet' if $facet; |
| 567 |
push @suffixes, 'suggestion' if $suggestible; |
| 568 |
push @suffixes, 'sort' if !defined $sort || $sort; |
| 569 |
|
| 570 |
foreach my $suffix (@suffixes) { |
| 571 |
my $mapping = ["${target_name}__$suffix"]; |
| 572 |
# TODO: Hack, fix later in less hideous manner |
| 573 |
if ($suffix eq 'suggestion') { |
| 574 |
push @{$mapping}, {%{$default_options}, property => 'input'}; |
| 447 |
} |
575 |
} |
| 448 |
my $default_options = {}; |
576 |
else { |
| 449 |
if ($substr_args) { |
577 |
push @{$mapping}, $default_options; |
| 450 |
$default_options->{substr} = $substr_args; |
|
|
| 451 |
} |
578 |
} |
|
|
579 |
push @mappings, $mapping; |
| 580 |
} |
| 581 |
return @mappings; |
| 582 |
}; |
| 452 |
|
583 |
|
| 453 |
# TODO: Should probably have per type value callback/hook |
584 |
=head2 _get_marc_mapping_rules |
| 454 |
# but hard code for now |
|
|
| 455 |
if ($target_type eq 'boolean') { |
| 456 |
$default_options->{value_callbacks} //= []; |
| 457 |
push @{$default_options->{value_callbacks}}, sub { |
| 458 |
my ($value) = @_; |
| 459 |
# Trim whitespace at both ends |
| 460 |
$value =~ s/^\s+|\s+$//g; |
| 461 |
return $value ? 'true' : 'false'; |
| 462 |
}; |
| 463 |
} |
| 464 |
|
585 |
|
| 465 |
my $mapping = [$target_name, $default_options]; |
586 |
my $mapping_rules = $self->_get_marc_mapping_rules() |
| 466 |
push @mappings, $mapping; |
|
|
| 467 |
|
587 |
|
| 468 |
my @suffixes = (); |
588 |
Generates rules from mappings stored in database for MARC records to Elasticsearch JSON document conversion. |
| 469 |
push @suffixes, 'facet' if $facet; |
589 |
|
| 470 |
push @suffixes, 'suggestion' if $suggestible; |
590 |
Since field retrieval is slow in C<MARC::Records> (all fields are itereted through for |
| 471 |
push @suffixes, 'sort' if !defined $sort || $sort; |
591 |
each call to C<MARC::Record>->field) we create an optimized structure of mapping |
|
|
592 |
rules keyed by MARC field tags holding all the mapping rules for that particular tag. |
| 593 |
|
| 594 |
We can then iterate through all MARC fields for each record and apply all relevant |
| 595 |
rules once per fields instead of retreiving fields multiple times for each mapping rule |
| 596 |
wich is terribly slow. |
| 597 |
|
| 598 |
=cut |
| 599 |
|
| 600 |
# TODO: This structure can be used for processing multiple MARC::Records so is currently |
| 601 |
# rebuilt for each batch. Since it is cacheable it could also be stored in an in |
| 602 |
# memory cache which it is currently not. The performance gain of caching |
| 603 |
# would probably be marginal, but to do this could be a further improvement. |
| 604 |
|
| 605 |
sub _get_marc_mapping_rules { |
| 606 |
my ($self) = @_; |
| 607 |
|
| 608 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 609 |
my @rules; |
| 472 |
|
610 |
|
| 473 |
foreach my $suffix (@suffixes) { |
|
|
| 474 |
my $mapping = ["${target_name}__$suffix"]; |
| 475 |
# Hack, fix later in less hideous manner |
| 476 |
if ($suffix eq 'suggestion') { |
| 477 |
push @{$mapping}, {%{$default_options}, property => 'input'}; |
| 478 |
} |
| 479 |
else { |
| 480 |
push @{$mapping}, $default_options; |
| 481 |
} |
| 482 |
push @mappings, $mapping; |
| 483 |
} |
| 484 |
return @mappings; |
| 485 |
}; |
| 486 |
my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
611 |
my $field_spec_regexp = qr/^([0-9]{3})([()0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
| 487 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
612 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
| 488 |
my $rules = { |
613 |
my $rules = { |
|
Lines 494-500
sub get_marc_mapping_rules {
Link Here
|
| 494 |
}; |
619 |
}; |
| 495 |
|
620 |
|
| 496 |
$self->_foreach_mapping(sub { |
621 |
$self->_foreach_mapping(sub { |
| 497 |
my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_; |
622 |
my ($name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field) = @_; |
| 498 |
return if $marc_type ne $marcflavour; |
623 |
return if $marc_type ne $marcflavour; |
| 499 |
|
624 |
|
| 500 |
if ($type eq 'sum') { |
625 |
if ($type eq 'sum') { |
|
Lines 550-556
sub get_marc_mapping_rules {
Link Here
|
| 550 |
} |
675 |
} |
| 551 |
|
676 |
|
| 552 |
my $range = defined $3 ? $3 : undef; |
677 |
my $range = defined $3 ? $3 : undef; |
| 553 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
678 |
my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
| 554 |
|
679 |
|
| 555 |
if ($field_tag < 10) { |
680 |
if ($field_tag < 10) { |
| 556 |
$rules->{control_fields}->{$field_tag} //= []; |
681 |
$rules->{control_fields}->{$field_tag} //= []; |
|
Lines 570-580
sub get_marc_mapping_rules {
Link Here
|
| 570 |
} |
695 |
} |
| 571 |
elsif ($marc_field =~ $leader_regexp) { |
696 |
elsif ($marc_field =~ $leader_regexp) { |
| 572 |
my $range = defined $1 ? $1 : undef; |
697 |
my $range = defined $1 ? $1 : undef; |
| 573 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
698 |
my @mappings = $self->_field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
| 574 |
push @{$rules->{leader}}, @mappings; |
699 |
push @{$rules->{leader}}, @mappings; |
| 575 |
} |
700 |
} |
| 576 |
else { |
701 |
else { |
| 577 |
die("Invalid marc field: $marc_field"); |
702 |
die("Invalid MARC field: $marc_field"); |
| 578 |
} |
703 |
} |
| 579 |
}); |
704 |
}); |
| 580 |
return $rules; |
705 |
return $rules; |