|
Lines 16-22
Link Here
|
| 16 |
# along with Koha; if not, see <https://www.gnu.org/licenses>. |
16 |
# along with Koha; if not, see <https://www.gnu.org/licenses>. |
| 17 |
|
17 |
|
| 18 |
use Modern::Perl; |
18 |
use Modern::Perl; |
| 19 |
use Encode; |
19 |
use utf8; |
| 20 |
|
20 |
|
| 21 |
use Test::More tests => 9; |
21 |
use Test::More tests => 9; |
| 22 |
use Test::NoWarnings; |
22 |
use Test::NoWarnings; |
|
Lines 211-217
subtest 'get_elasticsearch_mappings() tests' => sub {
Link Here
|
| 211 |
|
211 |
|
| 212 |
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub { |
212 |
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub { |
| 213 |
|
213 |
|
| 214 |
plan tests => 71; |
214 |
plan tests => 82; |
| 215 |
|
215 |
|
| 216 |
t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' ); |
216 |
t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' ); |
| 217 |
t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' ); |
217 |
t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' ); |
|
Lines 428-433
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 428 |
marc_type => 'marc21', |
428 |
marc_type => 'marc21', |
| 429 |
marc_field => '522a', |
429 |
marc_field => '522a', |
| 430 |
}, |
430 |
}, |
|
|
431 |
{ |
| 432 |
name => 'local-number', |
| 433 |
type => 'string', |
| 434 |
facet => 0, |
| 435 |
suggestible => 0, |
| 436 |
searchable => 1, |
| 437 |
sort => 1, |
| 438 |
marc_type => 'marc21', |
| 439 |
marc_field => '999c', |
| 440 |
}, |
| 431 |
); |
441 |
); |
| 432 |
|
442 |
|
| 433 |
my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch'); |
443 |
my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch'); |
|
Lines 460-466
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 460 |
my $long_callno = '1234567890' x 30; |
470 |
my $long_callno = '1234567890' x 30; |
| 461 |
|
471 |
|
| 462 |
my $marc_record_1 = MARC::Record->new(); |
472 |
my $marc_record_1 = MARC::Record->new(); |
| 463 |
$marc_record_1->leader(' cam 22 a 4500'); |
473 |
$marc_record_1->leader(' cam a22 a 4500'); |
| 464 |
$marc_record_1->append_fields( |
474 |
$marc_record_1->append_fields( |
| 465 |
MARC::Field->new( '001', '123' ), |
475 |
MARC::Field->new( '001', '123' ), |
| 466 |
MARC::Field->new( '007', 'ku' ), |
476 |
MARC::Field->new( '007', 'ku' ), |
|
Lines 481-488
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 481 |
MARC::Field->new( '952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2 ), |
491 |
MARC::Field->new( '952', '', '', 0 => 0, g => '127.20', o => $callno2, l => 2 ), |
| 482 |
MARC::Field->new( '952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1 ), |
492 |
MARC::Field->new( '952', '', '', 0 => 1, g => '0.00', o => $long_callno, l => 1 ), |
| 483 |
); |
493 |
); |
|
|
494 |
|
| 484 |
my $marc_record_2 = MARC::Record->new(); |
495 |
my $marc_record_2 = MARC::Record->new(); |
| 485 |
$marc_record_2->leader(' cam 22 a 4500'); |
496 |
$marc_record_2->leader(' cam a22 a 4500'); |
| 486 |
$marc_record_2->append_fields( |
497 |
$marc_record_2->append_fields( |
| 487 |
MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ), |
498 |
MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ), |
| 488 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
499 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
|
Lines 495-501
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 495 |
); |
506 |
); |
| 496 |
|
507 |
|
| 497 |
my $marc_record_3 = MARC::Record->new(); |
508 |
my $marc_record_3 = MARC::Record->new(); |
| 498 |
$marc_record_3->leader(' cam 22 a 4500'); |
509 |
$marc_record_3->leader(' cam a22 a 4500'); |
| 499 |
$marc_record_3->append_fields( |
510 |
$marc_record_3->append_fields( |
| 500 |
MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ), |
511 |
MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ), |
| 501 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
512 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
|
Lines 508-514
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 508 |
); |
519 |
); |
| 509 |
|
520 |
|
| 510 |
my $marc_record_4 = MARC::Record->new(); |
521 |
my $marc_record_4 = MARC::Record->new(); |
| 511 |
$marc_record_4->leader(' cam 22 a 4500'); |
522 |
$marc_record_4->leader(' cam a22 a 4500'); |
| 512 |
$marc_record_4->append_fields( |
523 |
$marc_record_4->append_fields( |
| 513 |
MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ), |
524 |
MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ), |
| 514 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
525 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
|
Lines 630-636
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 630 |
ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' ); |
641 |
ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' ); |
| 631 |
is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' ); |
642 |
is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' ); |
| 632 |
|
643 |
|
| 633 |
my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] ); |
644 |
my $decoded_marc_record = $see->search_document_marc_record_decode($docs->[0]); |
| 634 |
|
645 |
|
| 635 |
ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" ); |
646 |
ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" ); |
| 636 |
is( |
647 |
is( |
|
Lines 749-768
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 749 |
# Marc serialization format fallback for records exceeding base64ISO2709 max record size |
760 |
# Marc serialization format fallback for records exceeding base64ISO2709 max record size |
| 750 |
|
761 |
|
| 751 |
my $large_marc_record = MARC::Record->new(); |
762 |
my $large_marc_record = MARC::Record->new(); |
| 752 |
$large_marc_record->leader(' cam 22 a 4500'); |
763 |
$large_marc_record->leader(' cam a22 a 4500'); |
| 753 |
|
764 |
|
| 754 |
$large_marc_record->append_fields( |
765 |
$large_marc_record->append_fields( |
| 755 |
MARC::Field->new( '100', '', '', a => 'Author 1' ), |
766 |
MARC::Field->new( '100', '', '', a => 'Author 1' ), |
| 756 |
MARC::Field->new( '110', '', '', a => 'Corp Author' ), |
767 |
MARC::Field->new( '110', '', '', a => 'Corp Author' ), |
| 757 |
MARC::Field->new( '210', '', '', a => 'Title 1' ), |
768 |
MARC::Field->new( '210', '', '', a => 'Title 1' ), |
| 758 |
MARC::Field->new( '245', '', '', a => 'Title:', b => 'large record' ), |
769 |
# "|" is for testing escaping for multiple values with custom format |
| 759 |
MARC::Field->new( '999', '', '', c => '1234567' ), |
770 |
MARC::Field->new( '245', '', '', a => 'Title:', b => 'large | record' ), |
|
|
771 |
MARC::Field->new( '999', '', '', c => '1234569' ), |
| 760 |
); |
772 |
); |
| 761 |
|
773 |
|
| 762 |
my $item_field = MARC::Field->new( |
774 |
my $item_field = MARC::Field->new( |
| 763 |
'952', '', '', o => '123456789123456789123456789', p => '123456789', |
775 |
'952', '', '', o => '123456789123456789123456789', p => '123456789', |
| 764 |
z => Encode::decode( 'UTF-8', 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.' ) |
776 |
z => 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.' |
| 765 |
); |
777 |
); |
|
|
778 |
|
| 766 |
my $items_count = 1638; |
779 |
my $items_count = 1638; |
| 767 |
while ( --$items_count ) { |
780 |
while ( --$items_count ) { |
| 768 |
$large_marc_record->append_fields($item_field); |
781 |
$large_marc_record->append_fields($item_field); |
|
Lines 775-781
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 775 |
'For record exceeding max record size marc_format should be set correctly' |
788 |
'For record exceeding max record size marc_format should be set correctly' |
| 776 |
); |
789 |
); |
| 777 |
|
790 |
|
| 778 |
$decoded_marc_record = $see->decode_record_from_result( $docs->[0] ); |
791 |
$decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] ); |
| 779 |
|
792 |
|
| 780 |
ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" ); |
793 |
ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" ); |
| 781 |
is( |
794 |
is( |
|
Lines 783-788
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 783 |
"Decoded MARCXML record has same data as original record" |
796 |
"Decoded MARCXML record has same data as original record" |
| 784 |
); |
797 |
); |
| 785 |
|
798 |
|
|
|
799 |
# Search export functionality |
| 800 |
# Koha::SearchEngine::Elasticsearch::search_documents_encode() |
| 801 |
my @source_docs = ($marc_record_1, $marc_record_2, $large_marc_record); |
| 802 |
my @es_response_docs; |
| 803 |
my $records_data; |
| 804 |
|
| 805 |
for my $es_marc_format ('MARCXML', 'ARRAY', 'base64ISO2709') { |
| 806 |
|
| 807 |
t::lib::Mocks::mock_preference('ElasticsearchMARCFormat', $es_marc_format); |
| 808 |
|
| 809 |
$docs = $see->marc_records_to_documents(\@source_docs); |
| 810 |
|
| 811 |
# Emulate Elasticsearch response docs structure |
| 812 |
@es_response_docs = map { { _source => $_ } } @{$docs}; |
| 813 |
|
| 814 |
$records_data = $see->search_documents_encode(\@es_response_docs, 'ISO2709'); |
| 815 |
|
| 816 |
# $large_marc_record should not have been encoded as ISO2709 |
| 817 |
# since exceeds maximum size, see above |
| 818 |
my @tmp = ($marc_record_1, $marc_record_2); |
| 819 |
is( |
| 820 |
$records_data->{ISO2709}, |
| 821 |
join('', map { $_->as_usmarc() } @tmp), |
| 822 |
"ISO2709 encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\"" |
| 823 |
); |
| 824 |
|
| 825 |
my $expected_marc_xml = join("\n", |
| 826 |
MARC::File::XML::header(), |
| 827 |
MARC::File::XML::record($large_marc_record, 'MARC21'), |
| 828 |
MARC::File::XML::footer() |
| 829 |
); |
| 830 |
|
| 831 |
is( |
| 832 |
$records_data->{MARCXML}, |
| 833 |
$expected_marc_xml, |
| 834 |
"Record from search result encoded as MARCXML since exceeding ISO2709 maximum size is identical with source record using index format \"$es_marc_format\"" |
| 835 |
); |
| 836 |
|
| 837 |
$records_data = $see->search_documents_encode(\@es_response_docs, 'MARCXML'); |
| 838 |
|
| 839 |
$expected_marc_xml = join("\n", |
| 840 |
MARC::File::XML::header(), |
| 841 |
join("\n", map { MARC::File::XML::record($_, 'MARC21') } @source_docs), |
| 842 |
MARC::File::XML::footer() |
| 843 |
); |
| 844 |
|
| 845 |
is( |
| 846 |
$records_data->{MARCXML}, |
| 847 |
$expected_marc_xml, |
| 848 |
"MARCXML encoded records from Elasticsearch result are identical with source records using index format \"$es_marc_format\"" |
| 849 |
); |
| 850 |
} |
| 851 |
|
| 852 |
my $custom_formats = <<'END'; |
| 853 |
- name: Biblionumbers |
| 854 |
fields: [local-number] |
| 855 |
multiple: ignore |
| 856 |
- name: Title and author |
| 857 |
fields: [title, author] |
| 858 |
multiple: join |
| 859 |
END |
| 860 |
t::lib::Mocks::mock_preference('ElasticsearchSearchResultExportCustomFormats', $custom_formats); |
| 861 |
$custom_formats = C4::Context->yaml_preference('ElasticsearchSearchResultExportCustomFormats'); |
| 862 |
|
| 863 |
# Biblionumbers custom format |
| 864 |
$records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[0]); |
| 865 |
# UTF-8 encode? |
| 866 |
is( |
| 867 |
$records_data, |
| 868 |
"1234567\n1234568\n1234569", |
| 869 |
"Records where correctly encoded for the custom format \"Biblionumbers\"" |
| 870 |
); |
| 871 |
|
| 872 |
# Title and author custom format |
| 873 |
$records_data = $see->search_documents_custom_format_encode(\@es_response_docs, $custom_formats->[1]); |
| 874 |
|
| 875 |
my $encoded_data = join( |
| 876 |
"\n", |
| 877 |
"\"Title:|first record|Title: first record\",\"Author 1|Corp Author\"", |
| 878 |
"\"\",\"Author 2\"", |
| 879 |
"\"Title:|large \\| record|Title: large \\| record\",\"Author 1|Corp Author\"" |
| 880 |
); |
| 881 |
|
| 882 |
is( |
| 883 |
$records_data, |
| 884 |
$encoded_data, |
| 885 |
"Records where correctly encoded for the custom format \"Title and author\"" |
| 886 |
); |
| 887 |
|
| 786 |
push @mappings, { |
888 |
push @mappings, { |
| 787 |
name => 'title', |
889 |
name => 'title', |
| 788 |
type => 'string', |
890 |
type => 'string', |
|
Lines 826-832
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 826 |
|
928 |
|
| 827 |
pop @mappings; |
929 |
pop @mappings; |
| 828 |
my $marc_record_with_blank_field = MARC::Record->new(); |
930 |
my $marc_record_with_blank_field = MARC::Record->new(); |
| 829 |
$marc_record_with_blank_field->leader(' cam 22 a 4500'); |
931 |
$marc_record_with_blank_field->leader(' cam a22 a 4500'); |
| 830 |
|
932 |
|
| 831 |
$marc_record_with_blank_field->append_fields( |
933 |
$marc_record_with_blank_field->append_fields( |
| 832 |
MARC::Field->new( '100', '', '', a => '' ), |
934 |
MARC::Field->new( '100', '', '', a => '' ), |
|
Lines 839-845
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 839 |
is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' ); |
941 |
is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' ); |
| 840 |
|
942 |
|
| 841 |
my $marc_record_with_large_field = MARC::Record->new(); |
943 |
my $marc_record_with_large_field = MARC::Record->new(); |
| 842 |
$marc_record_with_large_field->leader(' cam 22 a 4500'); |
944 |
$marc_record_with_large_field->leader(' cam a22 a 4500'); |
| 843 |
|
945 |
|
| 844 |
my $xs = 'X' x 8191; |
946 |
my $xs = 'X' x 8191; |
| 845 |
my $ys = 'Y' x 8191; |
947 |
my $ys = 'Y' x 8191; |
|
Lines 872-878
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
Link Here
|
| 872 |
|
974 |
|
| 873 |
is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' ); |
975 |
is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' ); |
| 874 |
|
976 |
|
| 875 |
$decoded_marc_record = $see->decode_record_from_result( $docs->[0] ); |
977 |
$decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] ); |
| 876 |
|
978 |
|
| 877 |
ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" ); |
979 |
ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" ); |
| 878 |
is( |
980 |
is( |
|
Lines 928-934
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t
Link Here
|
| 928 |
{ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } ); |
1030 |
{ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } ); |
| 929 |
|
1031 |
|
| 930 |
my $marc_record_1 = MARC::Record->new(); |
1032 |
my $marc_record_1 = MARC::Record->new(); |
| 931 |
$marc_record_1->leader(' cam 22 a 4500'); |
1033 |
$marc_record_1->leader(' cam a22 a 4500'); |
| 932 |
$marc_record_1->append_fields( |
1034 |
$marc_record_1->append_fields( |
| 933 |
MARC::Field->new( '001', '123' ), |
1035 |
MARC::Field->new( '001', '123' ), |
| 934 |
MARC::Field->new( '020', '', '', a => '1-56619-909-3' ), |
1036 |
MARC::Field->new( '020', '', '', a => '1-56619-909-3' ), |
|
Lines 939-945
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t
Link Here
|
| 939 |
MARC::Field->new( '999', '', '', c => '1234567' ), |
1041 |
MARC::Field->new( '999', '', '', c => '1234567' ), |
| 940 |
); |
1042 |
); |
| 941 |
my $marc_record_2 = MARC::Record->new(); |
1043 |
my $marc_record_2 = MARC::Record->new(); |
| 942 |
$marc_record_2->leader(' cam 22 a 4500'); |
1044 |
$marc_record_2->leader(' cam a22 a 4500'); |
| 943 |
$marc_record_2->append_fields( |
1045 |
$marc_record_2->append_fields( |
| 944 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
1046 |
MARC::Field->new( '100', '', '', a => 'Author 2' ), |
| 945 |
|
1047 |
|
|
Lines 961-967
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () t
Link Here
|
| 961 |
|
1063 |
|
| 962 |
is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' ); |
1064 |
is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' ); |
| 963 |
|
1065 |
|
| 964 |
my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] ); |
1066 |
my $decoded_marc_record = $see->search_document_marc_record_decode( $docs->[0] ); |
| 965 |
|
1067 |
|
| 966 |
ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" ); |
1068 |
ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" ); |
| 967 |
is( |
1069 |
is( |
|
Lines 1148-1153
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu
Link Here
|
| 1148 |
MARC::Field->new( 150, '', '', a => 'Foo' ), |
1250 |
MARC::Field->new( 150, '', '', a => 'Foo' ), |
| 1149 |
MARC::Field->new( 450, '', '', a => 'Bar' ), |
1251 |
MARC::Field->new( 450, '', '', a => 'Bar' ), |
| 1150 |
); |
1252 |
); |
|
|
1253 |
$authority_record->encoding('UTF-8'); |
| 1254 |
|
| 1151 |
$dbh->do( |
1255 |
$dbh->do( |
| 1152 |
"INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef, |
1256 |
"INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef, |
| 1153 |
( $authority_record->as_xml_record('MARC21') ) |
1257 |
( $authority_record->as_xml_record('MARC21') ) |
|
Lines 1193-1199
subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with Inclu
Link Here
|
| 1193 |
{ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } ); |
1297 |
{ index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } ); |
| 1194 |
|
1298 |
|
| 1195 |
my $marc_record_1 = MARC::Record->new(); |
1299 |
my $marc_record_1 = MARC::Record->new(); |
| 1196 |
$marc_record_1->leader(' cam 22 a 4500'); |
1300 |
$marc_record_1->leader(' cam a22 a 4500'); |
| 1197 |
$marc_record_1->append_fields( |
1301 |
$marc_record_1->append_fields( |
| 1198 |
MARC::Field->new( '001', '123' ), |
1302 |
MARC::Field->new( '001', '123' ), |
| 1199 |
MARC::Field->new( '245', '', '', a => 'Title' ), |
1303 |
MARC::Field->new( '245', '', '', a => 'Title' ), |
| 1200 |
- |
|
|