Lines 34-39
use Search::Elasticsearch;
Link Here
|
34 |
use Try::Tiny; |
34 |
use Try::Tiny; |
35 |
use YAML::Syck; |
35 |
use YAML::Syck; |
36 |
|
36 |
|
|
|
37 |
use List::Util qw( sum0 ); |
38 |
use Search::Elasticsearch; |
39 |
use MARC::File::XML; |
40 |
|
37 |
__PACKAGE__->mk_ro_accessors(qw( index )); |
41 |
__PACKAGE__->mk_ro_accessors(qw( index )); |
38 |
__PACKAGE__->mk_accessors(qw( sort_fields )); |
42 |
__PACKAGE__->mk_accessors(qw( sort_fields )); |
39 |
|
43 |
|
Lines 67-72
sub new {
Link Here
|
67 |
return $self; |
71 |
return $self; |
68 |
} |
72 |
} |
69 |
|
73 |
|
|
|
74 |
sub get_elasticsearch { |
75 |
my $self = shift @_; |
76 |
unless (defined $self->{elasticsearch}) { |
77 |
my $conf = $self->get_elasticsearch_params(); |
78 |
$self->{elasticsearch} = Search::Elasticsearch->new( |
79 |
client => "5_0::Direct", |
80 |
nodes => $conf->{nodes}, |
81 |
cxn_pool => 'Sniff' |
82 |
); |
83 |
} |
84 |
return $self->{elasticsearch}; |
85 |
} |
86 |
|
70 |
=head2 get_elasticsearch_params |
87 |
=head2 get_elasticsearch_params |
71 |
|
88 |
|
72 |
my $params = $self->get_elasticsearch_params(); |
89 |
my $params = $self->get_elasticsearch_params(); |
Lines 281-286
sub sort_fields {
Link Here
|
281 |
return $self->_sort_fields_accessor(); |
298 |
return $self->_sort_fields_accessor(); |
282 |
} |
299 |
} |
283 |
|
300 |
|
|
|
301 |
sub marc_records_to_documents { |
302 |
my ($self, $records) = @_; |
303 |
my $rules = $self->get_marc_mapping_rules(); |
304 |
my $control_fields_rules = $rules->{control_fields}; |
305 |
my $data_fields_rules = $rules->{data_fields}; |
306 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
307 |
|
308 |
my @record_documents; |
309 |
|
310 |
sub _process_mappings { |
311 |
my ($mappings, $data, $record_document) = @_; |
312 |
foreach my $mapping (@{$mappings}) { |
313 |
my ($target, $options) = @{$mapping}; |
314 |
my $_data = $data; |
315 |
$record_document->{$target} //= []; |
316 |
if ($options->{substr}) { |
317 |
my ($offset, $length) = @{$options->{substr}}; |
318 |
$_data = substr $data, $offset, $length; |
319 |
} |
320 |
if ($options->{property}) { |
321 |
$_data = { |
322 |
$options->{property} => $_data |
323 |
} |
324 |
} |
325 |
push @{$record_document->{$target}}, $_data; |
326 |
} |
327 |
} |
328 |
foreach my $record (@{$records}) { |
329 |
my $record_document = {}; |
330 |
my $mappings = $rules->{leader}; |
331 |
if ($mappings) { |
332 |
_process_mappings($mappings, $record->leader(), $record_document); |
333 |
} |
334 |
foreach my $field ($record->fields()) { |
335 |
if($field->is_control_field()) { |
336 |
my $mappings = $control_fields_rules->{$field->tag()}; |
337 |
if ($mappings) { |
338 |
_process_mappings($mappings, $field->data(), $record_document); |
339 |
} |
340 |
} |
341 |
else { |
342 |
my $subfields_mappings = $data_fields_rules->{$field->tag()}; |
343 |
if ($subfields_mappings) { |
344 |
my $wildcard_mappings = $subfields_mappings->{'*'}; |
345 |
foreach my $subfield ($field->subfields()) { |
346 |
my ($code, $data) = @{$subfield}; |
347 |
my $mappings = $subfields_mappings->{$code} // []; |
348 |
if ($wildcard_mappings) { |
349 |
$mappings = [@{$mappings}, @{$wildcard_mappings}]; |
350 |
} |
351 |
if (@{$mappings}) { |
352 |
_process_mappings($mappings, $data, $record_document); |
353 |
} |
354 |
} |
355 |
} |
356 |
} |
357 |
} |
358 |
foreach my $field (keys %{$rules->{defaults}}) { |
359 |
unless (defined $record_document->{$field}) { |
360 |
$record_document->{$field} = $rules->{defaults}->{$field}; |
361 |
} |
362 |
} |
363 |
foreach my $field (@{$rules->{sum}}) { |
364 |
if (defined $record_document->{$field}) { |
365 |
# TODO: validate numeric? filter? |
366 |
# TODO: Or should only accept fields without nested values? |
367 |
# TODO: Quick and dirty, improve if needed |
368 |
$record_document->{$field} = sum0(grep { ref($_) eq 'SCALAR' && m/\d+([.,]\d+)?/} @{$record_document->{$field}}); |
369 |
} |
370 |
} |
371 |
# TODO: Perhaps should check if $records_document non empty, but really should never be the case |
372 |
$record->encoding('UTF-8'); |
373 |
$record_document->{'marc_xml'} = $record->as_xml_record($marcflavour); |
374 |
my $id = $record->subfield('999', 'c'); |
375 |
push @record_documents, [$id, $record_document]; |
376 |
} |
377 |
return \@record_documents; |
378 |
} |
379 |
|
380 |
# Provides the rules for marc to Elasticsearch JSON document conversion. |
381 |
sub get_marc_mapping_rules { |
382 |
my ($self) = @_; |
383 |
|
384 |
my $marcflavour = lc C4::Context->preference('marcflavour'); |
385 |
my @rules; |
386 |
|
387 |
sub _field_mappings { |
388 |
my ($facet, $suggestible, $sort, $target_name, $target_type, $range) = @_; |
389 |
my %mapping_defaults = (); |
390 |
my @mappings; |
391 |
|
392 |
my $substr_args = undef; |
393 |
if ($range) { |
394 |
my ($offset, $end) = map(int, split /-/, $range, 2); |
395 |
$substr_args = [$offset]; |
396 |
push @{$substr_args}, (defined $end ? $end - $offset : 1); |
397 |
} |
398 |
my $default_options = {}; |
399 |
if ($substr_args) { |
400 |
$default_options->{substr} = $substr_args; |
401 |
} |
402 |
|
403 |
my $mapping = [$target_name, $default_options]; |
404 |
push @mappings, $mapping; |
405 |
|
406 |
my @suffixes = (); |
407 |
push @suffixes, 'facet' if $facet; |
408 |
push @suffixes, 'suggestion' if $suggestible; # Check condition, also if undef? |
409 |
push @suffixes, 'sort' if $sort; |
410 |
foreach my $suffix (@suffixes) { |
411 |
my $mapping = ["${target_name}__$suffix"]; |
412 |
# Hack, fix later in less hideous manner |
413 |
if ($suffix eq 'suggestion') { |
414 |
push @{$mapping}, {%{$default_options}, property => 'input'}; |
415 |
} |
416 |
else { |
417 |
push @{$mapping}, $default_options; |
418 |
} |
419 |
push @mappings, $mapping; |
420 |
} |
421 |
return @mappings; |
422 |
}; |
423 |
my $field_spec_regexp = qr/^([0-9]{3})([0-9a-z]+)?(?:_\/(\d+(?:-\d+)?))?$/; |
424 |
my $leader_regexp = qr/^leader(?:_\/(\d+(?:-\d+)?))?$/; |
425 |
my $rules = { |
426 |
'leader' => [], |
427 |
'control_fields' => {}, |
428 |
'data_fields' => {}, |
429 |
'sum' => [], |
430 |
'defaults' => {} |
431 |
}; |
432 |
|
433 |
$self->_foreach_mapping(sub { |
434 |
my ( $name, $type, $facet, $suggestible, $sort, $marc_type, $marc_field ) = @_; |
435 |
return if $marc_type ne $marcflavour; |
436 |
|
437 |
if ($type eq 'sum') { |
438 |
push @{$rules->{sum}}, $name; |
439 |
} |
440 |
elsif($type eq 'boolean') { |
441 |
# boolean gets special handling, if value doesn't exist for a field, |
442 |
# it is set to false |
443 |
$rules->{defaults}->{$name} = 0; |
444 |
} |
445 |
|
446 |
if ($marc_field =~ $field_spec_regexp) { |
447 |
my $field_tag = $1; |
448 |
my $subfields = defined $2 ? $2 : '*'; |
449 |
my $range = defined $3 ? $3 : undef; |
450 |
if ($field_tag < 10) { |
451 |
$rules->{control_fields}->{$field_tag} //= []; |
452 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
453 |
push @{$rules->{control_fields}->{$field_tag}}, @mappings; |
454 |
} |
455 |
else { |
456 |
$rules->{data_fields}->{$field_tag} //= {}; |
457 |
foreach my $subfield (split //, $subfields) { |
458 |
$rules->{data_fields}->{$field_tag}->{$subfield} //= []; |
459 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
460 |
push @{$rules->{data_fields}->{$field_tag}->{$subfield}}, @mappings; |
461 |
} |
462 |
} |
463 |
} |
464 |
elsif ($marc_field =~ $leader_regexp) { |
465 |
my $range = defined $1 ? $1 : undef; |
466 |
my @mappings = _field_mappings($facet, $suggestible, $sort, $name, $type, $range); |
467 |
push @{$rules->{leader}}, @mappings; |
468 |
} |
469 |
else { |
470 |
die("Invalid marc field: $marc_field"); |
471 |
} |
472 |
}); |
473 |
return $rules; |
474 |
} |
475 |
|
284 |
# Provides the rules for data conversion. |
476 |
# Provides the rules for data conversion. |
285 |
sub get_fixer_rules { |
477 |
sub get_fixer_rules { |
286 |
my ($self) = @_; |
478 |
my ($self) = @_; |