Attachment #79012 for bug #20244

View | Details | Raw Unified | Return to bug 20244
Collapse All | Expand All




use MARC::File::XML;
use MIME::Base64;
use Encode qw(encode);
use Business::ISBN;

__PACKAGE__->mk_ro_accessors(qw( index ));
__PACKAGE__->mk_accessors(qw( sort_fields ));

                $record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
            }
        }
        # Index all applicable ISBN forms (ISBN-10 and ISBN-13 with and without dashes)
        foreach my $field (@{$rules->{isbn}}) {
            if (defined $record_document->{$field}) {
                my @isbns = ();
                foreach my $input_isbn (@{$record_document->{$field}}) {
                    my $isbn = Business::ISBN->new($input_isbn);
                    if (defined $isbn && $isbn->is_valid) {
                        my $isbn13 = $isbn->as_isbn13->as_string;
                        push @isbns, $isbn13;
                        $isbn13 =~ s/\-//g;
                        push @isbns, $isbn13;

                        my $isbn10 = $isbn->as_isbn10;
                        if ($isbn10) {
                            $isbn10 = $isbn10->as_string;
                            push @isbns, $isbn10;
                            $isbn10 =~ s/\-//g;
                            push @isbns, $isbn10;
                        }
                    } else {
                        push @isbns, $input_isbn;
                    }
                }
                $record_document->{$field} = \@isbns;
            }
        }

        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
        $record->encoding('UTF-8');
        my @warnings;

        'control_fields' => {},
        'data_fields' => {},
        'sum' => [],
        'isbn' => [],
        'defaults' => {}
    };


        if ($type eq 'sum') {
            push @{$rules->{sum}}, $name;
        }
        elsif ($type eq 'isbn') {
            push @{$rules->{isbn}}, $name;
        }
        elsif ($type eq 'boolean') {
            # boolean gets special handling, if value doesn't exist for a field,
            # it is set to false
            $rules->{defaults}->{$name} = 'false';





subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {

    plan tests => 32;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');

    my @mappings = (
        {
            name => 'isbn',
            type => 'isbn',
            facet => 0,
            suggestible => 0,
            sort => 0,
            marc_type => 'marc21',
            marc_field => '020a',
        },
        {
            name => 'author',
            type => 'string',
            facet => 1,

    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->leader('     cam  22      a 4500');
    $marc_record_1->append_fields(
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
        MARC::Field->new('100', '', '', a => 'Author 1'),
        MARC::Field->new('110', '', '', a => 'Corp Author'),
        MARC::Field->new('210', '', '', a => 'Title 1'),

    my $docs = $see->marc_records_to_documents($records);

    # First record:

    is(scalar @{$docs}, 2, 'Two records converted to documents');

    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');

        'First document type_of_record_and_bib_level field should be set correctly'
    );

    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');

    # Second record:

    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
- 

Return to bug 20244

Lines 39-44 use Search::Elasticsearch; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-1 / +33 lines)
39	use MARC::File::XML;	39	use MARC::File::XML;
40	use MIME::Base64;	40	use MIME::Base64;
41	use Encode qw(encode);	41	use Encode qw(encode);
		42	use Business::ISBN;
42		43
43	__PACKAGE__->mk_ro_accessors(qw( index ));	44	__PACKAGE__->mk_ro_accessors(qw( index ));
44	__PACKAGE__->mk_accessors(qw( sort_fields ));	45	__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 397-402 sub marc_records_to_documents { Link Here
397	$record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});	398	$record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
398	}	399	}
399	}	400	}
		401	# Index all applicable ISBN forms (ISBN-10 and ISBN-13 with and without dashes)
		402	foreach my $field (@{$rules->{isbn}}) {
		403	if (defined $record_document->{$field}) {
		404	my @isbns = ();
		405	foreach my $input_isbn (@{$record_document->{$field}}) {
		406	my $isbn = Business::ISBN->new($input_isbn);
		407	if (defined $isbn && $isbn->is_valid) {
		408	my $isbn13 = $isbn->as_isbn13->as_string;
		409	push @isbns, $isbn13;
		410	$isbn13 =~ s/\-//g;
		411	push @isbns, $isbn13;
		412
		413	my $isbn10 = $isbn->as_isbn10;
		414	if ($isbn10) {
		415	$isbn10 = $isbn10->as_string;
		416	push @isbns, $isbn10;
		417	$isbn10 =~ s/\-//g;
		418	push @isbns, $isbn10;
		419	}
		420	} else {
		421	push @isbns, $input_isbn;
		422	}
		423	}
		424	$record_document->{$field} = \@isbns;
		425	}
		426	}
		427
400	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	428	# TODO: Perhaps should check if $records_document non empty, but really should never be the case
401	$record->encoding('UTF-8');	429	$record->encoding('UTF-8');
402	my @warnings;	430	my @warnings;
Lines 490-495 sub get_marc_mapping_rules { Link Here
490	'control_fields' => {},	518	'control_fields' => {},
491	'data_fields' => {},	519	'data_fields' => {},
492	'sum' => [],	520	'sum' => [],
		521	'isbn' => [],
493	'defaults' => {}	522	'defaults' => {}
494	};	523	};
495		524
Lines 500-506 sub get_marc_mapping_rules { Link Here
500	if ($type eq 'sum') {	529	if ($type eq 'sum') {
501	push @{$rules->{sum}}, $name;	530	push @{$rules->{sum}}, $name;
502	}	531	}
503	elsif($type eq 'boolean') {	532	elsif ($type eq 'isbn') {
		533	push @{$rules->{isbn}}, $name;
		534	}
		535	elsif ($type eq 'boolean') {
504	# boolean gets special handling, if value doesn't exist for a field,	536	# boolean gets special handling, if value doesn't exist for a field,
505	# it is set to false	537	# it is set to false
506	$rules->{defaults}->{$name} = 'false';	538	$rules->{defaults}->{$name} = 'false';

Lines 115-126 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-3 / +14 lines)
115		115
116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117		117
118	plan tests => 30;	118	plan tests => 32;
119		119
120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121		121
122	my @mappings = (	122	my @mappings = (
123	{	123	{
		124	name => 'isbn',
		125	type => 'isbn',
		126	facet => 0,
		127	suggestible => 0,
		128	sort => 0,
		129	marc_type => 'marc21',
		130	marc_field => '020a',
		131	},
		132	{
124	name => 'author',	133	name => 'author',
125	type => 'string',	134	type => 'string',
126	facet => 1,	135	facet => 1,
Lines 225-230 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
225	my $marc_record_1 = MARC::Record->new();	234	my $marc_record_1 = MARC::Record->new();
226	$marc_record_1->leader(' cam 22 a 4500');	235	$marc_record_1->leader(' cam 22 a 4500');
227	$marc_record_1->append_fields(	236	$marc_record_1->append_fields(
		237	MARC::Field->new('020', '', '', a => '1-56619-909-3'),
228	MARC::Field->new('100', '', '', a => 'Author 1'),	238	MARC::Field->new('100', '', '', a => 'Author 1'),
229	MARC::Field->new('110', '', '', a => 'Corp Author'),	239	MARC::Field->new('110', '', '', a => 'Corp Author'),
230	MARC::Field->new('210', '', '', a => 'Title 1'),	240	MARC::Field->new('210', '', '', a => 'Title 1'),
Lines 250-256 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
250	my $docs = $see->marc_records_to_documents($records);	260	my $docs = $see->marc_records_to_documents($records);
251		261
252	# First record:	262	# First record:
253
254	is(scalar @{$docs}, 2, 'Two records converted to documents');	263	is(scalar @{$docs}, 2, 'Two records converted to documents');
255		264
256	is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');	265	is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
Lines 329-334 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
329	'First document type_of_record_and_bib_level field should be set correctly'	338	'First document type_of_record_and_bib_level field should be set correctly'
330	);	339	);
331		340
		341	is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
		342	is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
		343
332	# Second record:	344	# Second record:
333		345
334	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');	346	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
335	-