|
Lines 34-39
use Search::Elasticsearch;
Link Here
|
| 34 |
use Try::Tiny; |
34 |
use Try::Tiny; |
| 35 |
use YAML::Syck; |
35 |
use YAML::Syck; |
| 36 |
|
36 |
|
|
|
37 |
use List::Util qw( sum0 reduce ); |
| 38 |
use Search::Elasticsearch; |
| 39 |
use MARC::File::XML; |
| 40 |
use MIME::Base64; |
| 41 |
use Encode qw(encode); |
| 42 |
|
| 37 |
__PACKAGE__->mk_ro_accessors(qw( index )); |
43 |
__PACKAGE__->mk_ro_accessors(qw( index )); |
| 38 |
__PACKAGE__->mk_accessors(qw( sort_fields )); |
44 |
__PACKAGE__->mk_accessors(qw( sort_fields )); |
| 39 |
|
45 |
|
|
Lines 67-72
sub new {
Link Here
|
| 67 |
return $self; |
73 |
return $self; |
| 68 |
} |
74 |
} |
| 69 |
|
75 |
|
|
|
76 |
sub get_elasticsearch { |
| 77 |
my $self = shift @_; |
| 78 |
unless (defined $self->{elasticsearch}) { |
| 79 |
my $conf = $self->get_elasticsearch_params(); |
| 80 |
$self->{elasticsearch} = Search::Elasticsearch->new( |
| 81 |
client => "5_0::Direct", |
| 82 |
nodes => $conf->{nodes}, |
| 83 |
cxn_pool => 'Sniff' |
| 84 |
); |
| 85 |
} |
| 86 |
return $self->{elasticsearch}; |
| 87 |
} |
| 88 |
|
| 70 |
=head2 get_elasticsearch_params |
89 |
=head2 get_elasticsearch_params |
| 71 |
|
90 |
|
| 72 |
my $params = $self->get_elasticsearch_params(); |
91 |
my $params = $self->get_elasticsearch_params(); |
|
Lines 281-328
sub sort_fields {
Link Here
|
| 281 |
return $self->_sort_fields_accessor(); |
300 |
return $self->_sort_fields_accessor(); |
| 282 |
} |
301 |
} |
| 283 |
|
302 |
|
| 284 |
# Provides the rules for data conversion. |
303 |
sub marc_records_to_documents { |
| 285 |
sub get_fixer_rules { |
304 |
my ($self, $records) = @_; |
|
|
305 |
my $rules = $self->get_marc_mapping_rules(); |
| 306 |
my $control_fields_rules = $rules->{control_fields}; |
| 307 |
my $data_fields_rules = $rules->{data_fields}; |
| 308 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 309 |
my $serialization_format = C4::Context->preference('ElasticsearchMARCSerializationFormat'); |
| 310 |
|
| 311 |
my @record_documents; |
| 312 |
|
| 313 |
sub _process_mappings { |
| 314 |
my ($mappings, $data, $record_document) = @_; |
| 315 |
foreach my $mapping (@{$mappings}) { |
| 316 |
my ($target, $options) = @{$mapping}; |
| 317 |
# Copy (scalar) data since can have multiple targets |
| 318 |
# with differing options for (possibly) mutating data |
| 319 |
# so need a different copy for each |
| 320 |
my $_data = $data; |
| 321 |
$record_document->{$target} //= []; |
| 322 |
if (defined $options->{substr}) { |
| 323 |
my ($start, $length) = @{$options->{substr}}; |
| 324 |
$_data = length($data) > $start ? substr $data, $start, $length : ''; |
| 325 |
} |
| 326 |
if (defined $options->{value_callbacks}) { |
| 327 |
$_data = reduce { $b->($a) } ($_data, @{$options->{value_callbacks}}); |
| 328 |
} |
| 329 |
if (defined $options->{property}) { |
| 330 |
$_data = { |
| 331 |
$options->{property} => $_data |
| 332 |
} |
| 333 |
} |
| 334 |
push @{$record_document->{$target}}, $_data; |
| 335 |
} |
| 336 |
} |
| 337 |
foreach my $record (@{$records}) { |
| 338 |
my $record_document = {}; |
| 339 |
my $mappings = $rules->{leader}; |
| 340 |
if ($mappings) { |
| 341 |
_process_mappings($mappings, $record->leader(), $record_document); |
| 342 |
} |
| 343 |
foreach my $field ($record->fields()) { |
| 344 |
if($field->is_control_field()) { |
| 345 |
my $mappings = $control_fields_rules->{$field->tag()}; |
| 346 |
if ($mappings) { |
| 347 |
_process_mappings($mappings, $field->data(), $record_document); |
| 348 |
} |
| 349 |
} |
| 350 |
else { |
| 351 |
my $subfields_mappings = $data_fields_rules->{$field->tag()}; |
| 352 |
if ($subfields_mappings) { |
| 353 |
my $wildcard_mappings = $subfields_mappings->{'*'}; |
| 354 |
foreach my $subfield ($field->subfields()) { |
| 355 |
my ($code, $data) = @{$subfield}; |
| 356 |
my $mappings = $subfields_mappings->{$code} // []; |
| 357 |
if ($wildcard_mappings) { |
| 358 |
$mappings = [@{$mappings}, @{$wildcard_mappings}]; |
| 359 |
} |
| 360 |
if (@{$mappings}) { |
| 361 |
_process_mappings($mappings, $data, $record_document); |
| 362 |
} |
| 363 |
} |
| 364 |
} |
| 365 |
} |
| 366 |
} |
| 367 |
foreach my $field (keys %{$rules->{defaults}}) { |
| 368 |
unless (defined $record_document->{$field}) { |
| 369 |
$record_document->{$field} = $rules->{defaults}->{$field}; |
| 370 |
} |
| 371 |
} |
| 372 |
foreach my $field (@{$rules->{sum}}) { |
| 373 |
if (defined $record_document->{$field}) { |
| 374 |
# TODO: validate numeric? filter? |
| 375 |
# TODO: Or should only accept fields without nested values? |
| 376 |
# TODO: Quick and dirty, improve if needed |
| 377 |
$record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}}); |
| 378 |
} |
| 379 |
} |
| 380 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
| 381 |
$record->encoding('UTF-8'); |
| 382 |
if ($serialization_format eq 'base64ISO2709') { |
| 383 |
$record_document->{'marc_data'} = encode_base64(encode('UTF-8', $record->as_usmarc())); |
| 384 |
} |
| 385 |
else { |
| 386 |
$record_document->{'marc_data'} = $record->as_xml_record($marcflavour); |
| 387 |
} |
| 388 |
my $id = $record->subfield('999', 'c'); |
| 389 |
push @record_documents, [$id, $record_document]; |
| 390 |
} |
| 391 |
return \@record_documents; |
| 392 |
} |
| 393 |
|
| 394 |
# Provides the rules for marc to Elasticsearch JSON document conversion. |
| 395 |
sub get_marc_mapping_rules { |
| 286 |
my ($self) = @_; |
396 |
my ($self) = @_; |
| 287 |
|
397 |
|
| 288 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
398 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
| 289 |
my @rules; |
399 |
my @rules; |
| 290 |
|
400 |
|
| 291 |
$self->_foreach_mapping( |
401 |
sub _field_mappings { |
| 292 |
sub { |
402 |
my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_; |
| 293 |
my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_; |
403 |
my %mapping_defaults = (); |
| 294 |
return if $marc_type ne $marcflavour; |
404 |
my @mappings; |
| 295 |
my $options =''; |
405 |
|
|
|
406 |
my $substr_args = undef; |
| 407 |
if ($range) { |
| 408 |
# TODO: use value_callback instead? |
| 409 |
my ($start, $end) = map(int, split /-/, $range, 2); |
| 410 |
$substr_args = [$start]; |
| 411 |
push @{$substr_args}, (defined $end ? $end - $start + 1 : 1); |
| 412 |
} |
| 413 |
my $default_options = {}; |
| 414 |
if ($substr_args) { |
| 415 |
$default_options->{substr} = $substr_args; |
| 416 |
} |
| 417 |
|
| 418 |
# TODO: Should probably have per type value callback/hook |
| 419 |
# but hard code for now |
| 420 |
if ($target_type eq 'boolean') { |
| 421 |
$default_options->{value_callbacks} //= []; |
| 422 |
push @{$default_options->{value_callbacks}}, sub { |
| 423 |
my ($value) = @_; |
| 424 |
# Trim whitespace at both ends |
| 425 |
$value =~ s/^\s+|\s+$//g; |
| 426 |
return $value ? 'true' : 'false'; |
| 427 |
}; |
| 428 |
} |
| 296 |
|
429 |
|
| 297 |
push @rules, "marc_map('$marc_field','${name}.\$append', $options)"; |
430 |
my $mapping = [$target_name, $default_options]; |
| 298 |
if ($facet) { |
431 |
push @mappings, $mapping; |
| 299 |
push @rules, "marc_map('$marc_field','${name}__facet.\$append', $options)"; |
432 |
|
|
|
433 |
my @suffixes = (); |
| 434 |
push @suffixes, 'facet' if $facet; |
| 435 |
push @suffixes, 'suggestion' if $suggestible; |
| 436 |
push @suffixes, 'sort' if !defined $sort || $sort; |
| 437 |
|
| 438 |
foreach my $suffix (@suffixes) { |
| 439 |
my $mapping = ["${target_name}__$suffix"]; |
| 440 |
# Hack, fix later in less hideous manner |
| 441 |
if ($suffix eq 'suggestion') { |
| 442 |
push @{$mapping}, {%{$default_options}, property => 'input'}; |
| 300 |
} |
443 |
} |
| 301 |
if ($suggestible) { |
444 |
else { |
| 302 |
push @rules, |
445 |
push @{$mapping}, $default_options; |
| 303 |
#"marc_map('$marc_field','${name}__suggestion.input.\$append', '')"; #must not have nested data structures in .input |
|
|
| 304 |
"marc_map('$marc_field','${name}__suggestion.input.\$append')"; |
| 305 |
} |
446 |
} |
| 306 |
if ( $type eq 'boolean' ) { |
447 |
push @mappings, $mapping; |
|
|
448 |
} |
| 449 |
return @mappings; |
| 450 |
}; |
| 451 |
my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
| 452 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
| 453 |
my $rules = { |
| 454 |
'leader' => [], |
| 455 |
'control_fields' => {}, |
| 456 |
'data_fields' => {}, |
| 457 |
'sum' => [], |
| 458 |
'defaults' => {} |
| 459 |
}; |
| 460 |
|
| 461 |
$self->_foreach_mapping(sub { |
| 462 |
my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_; |
| 463 |
return if $marc_type ne $marcflavour; |
| 464 |
|
| 465 |
if ($type eq 'sum') { |
| 466 |
push @{$rules->{sum}}, $name; |
| 467 |
} |
| 468 |
elsif($type eq 'boolean') { |
| 469 |
# boolean gets special handling, if value doesn't exist for a field, |
| 470 |
# it is set to false |
| 471 |
$rules->{defaults}->{$name} = 'false'; |
| 472 |
} |
| 307 |
|
473 |
|
| 308 |
# boolean gets special handling, basically if it doesn't exist, |
474 |
if ($marc_field =~ $field_spec_regexp) { |
| 309 |
# it's added and set to false. Otherwise we can't query it. |
475 |
my $field_tag = $1; |
| 310 |
push @rules, |
476 |
my $subfields = defined $2 ? $2 : '*'; |
| 311 |
"unless exists('$name') add_field('$name', 0) end"; |
477 |
my $range = defined $3 ? $3 : undef; |
| 312 |
} |
478 |
if ($field_tag < 10) { |
| 313 |
if ($type eq 'sum' ) { |
479 |
$rules->{control_fields}->{$field_tag} //= []; |
| 314 |
push @rules, "sum('$name')"; |
480 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
|
|
481 |
push @{$rules->{control_fields}->{$field_tag}}, @mappings; |
| 315 |
} |
482 |
} |
| 316 |
if ($self->sort_fields()->{$name}) { |
483 |
else { |
| 317 |
if ($sort || !defined $sort) { |
484 |
$rules->{data_fields}->{$field_tag} //= {}; |
| 318 |
push @rules, "marc_map('$marc_field','${name}__sort.\$append', $options)"; |
485 |
foreach my $subfield (split //, $subfields) { |
|
|
486 |
$rules->{data_fields}->{$field_tag}->{$subfield} //= []; |
| 487 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
| 488 |
push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings; |
| 319 |
} |
489 |
} |
| 320 |
} |
490 |
} |
| 321 |
} |
491 |
} |
| 322 |
); |
492 |
elsif ($marc_field =~ $leader_regexp) { |
| 323 |
|
493 |
my $range = defined $1 ? $1 : undef; |
| 324 |
push @rules, "move_field(_id,es_id)"; #Also you must set the Catmandu::Store::ElasticSearch->new(key_prefix: 'es_'); |
494 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
| 325 |
return \@rules; |
495 |
push @{$rules->{leader}}, @mappings; |
|
|
496 |
} |
| 497 |
else { |
| 498 |
die("Invalid marc field: $marc_field"); |
| 499 |
} |
| 500 |
}); |
| 501 |
return $rules; |
| 326 |
} |
502 |
} |
| 327 |
|
503 |
|
| 328 |
=head2 _foreach_mapping |
504 |
=head2 _foreach_mapping |