Attachment #76646 for bug #20244

View | Details | Raw Unified | Return to bug 20244
Collapse All | Expand All




use MARC::File::XML;
use MIME::Base64;
use Encode qw(encode);
use Business::ISBN;

__PACKAGE__->mk_ro_accessors(qw( index ));
__PACKAGE__->mk_accessors(qw( sort_fields ));

                $record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
            }
        }
        # Index all applicable ISBN forms (ISBN-10 and ISBN-13 with and without dashes)
        foreach my $field (@{$rules->{isbn}}) {
            if (defined $record_document->{$field}) {
                my @isbns = ();
                foreach my $input_isbn (@{$record_document->{$field}}) {
                    my $isbn = Business::ISBN->new($input_isbn);
                    if (defined $isbn && $isbn->is_valid) {
                        my $isbn13 = $isbn->as_isbn13->as_string;
                        push @isbns, $isbn13;
                        $isbn13 =~ s/\-//g;
                        push @isbns, $isbn13;
                        
                        my $isbn10 = $isbn->as_isbn10;
                        if ($isbn10) {
                            $isbn10 = $isbn10->as_string;
                            push @isbns, $isbn10;
                            $isbn10 =~ s/\-//g;
                            push @isbns, $isbn10;
                        }
                    } else {
                        push @isbns, $input_isbn;
                    }                
                }
                $record_document->{$field} = \@isbns;
            }
        }

        # TODO: Perhaps should check if $records_document non empty, but really should never be the case
        $record->encoding('UTF-8');
        my @warnings;

        'control_fields' => {},
        'data_fields' => {},
        'sum' => [],
        'isbn' => [],
        'defaults' => {}
    };


        if ($type eq 'sum') {
            push @{$rules->{sum}}, $name;
        }
        elsif ($type eq 'isbn') {
            push @{$rules->{isbn}}, $name;
        }
        elsif ($type eq 'boolean') {
            # boolean gets special handling, if value doesn't exist for a field,
            # it is set to false
            $rules->{defaults}->{$name} = 'false';





subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {

    plan tests => 32;

    t::lib::Mocks::mock_preference('marcflavour', 'MARC21');

    my @mappings = (
        {
            name => 'isbn',
            type => 'isbn',
            facet => 0,
            suggestible => 0,
            sort => 0,
            marc_type => 'marc21',
            marc_field => '020a',
        },
        {
            name => 'author',
            type => 'string',
            facet => 1,

    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->leader('     cam  22      a 4500');
    $marc_record_1->append_fields(
        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
        MARC::Field->new('100', '', '', a => 'Author 1'),
        MARC::Field->new('110', '', '', a => 'Corp Author'),
        MARC::Field->new('210', '', '', a => 'Title 1'),

    my $docs = $see->marc_records_to_documents($records);

    # First record:

    is(scalar @{$docs}, 2, 'Two records converted to documents');

    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');    

    is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
    is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');

        'First document type_of_record_and_bib_level field should be set correctly'
    );

    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');

    # Second record:

    is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
- 

Return to bug 20244

Lines 39-44 use Search::Elasticsearch; Link Here

(-)a/Koha/SearchEngine/Elasticsearch.pm (-1 / +33 lines)
39	use MARC::File::XML;	39	use MARC::File::XML;
40	use MIME::Base64;	40	use MIME::Base64;
41	use Encode qw(encode);	41	use Encode qw(encode);
		42	use Business::ISBN;
42		43
43	__PACKAGE__->mk_ro_accessors(qw( index ));	44	__PACKAGE__->mk_ro_accessors(qw( index ));
44	__PACKAGE__->mk_accessors(qw( sort_fields ));	45	__PACKAGE__->mk_accessors(qw( sort_fields ));
Lines 377-382 sub marc_records_to_documents { Link Here
377	$record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});	378	$record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
378	}	379	}
379	}	380	}
		381	# Index all applicable ISBN forms (ISBN-10 and ISBN-13 with and without dashes)
		382	foreach my $field (@{$rules->{isbn}}) {
		383	if (defined $record_document->{$field}) {
		384	my @isbns = ();
		385	foreach my $input_isbn (@{$record_document->{$field}}) {
		386	my $isbn = Business::ISBN->new($input_isbn);
		387	if (defined $isbn && $isbn->is_valid) {
		388	my $isbn13 = $isbn->as_isbn13->as_string;
		389	push @isbns, $isbn13;
		390	$isbn13 =~ s/\-//g;
		391	push @isbns, $isbn13;
		392
		393	my $isbn10 = $isbn->as_isbn10;
		394	if ($isbn10) {
		395	$isbn10 = $isbn10->as_string;
		396	push @isbns, $isbn10;
		397	$isbn10 =~ s/\-//g;
		398	push @isbns, $isbn10;
		399	}
		400	} else {
		401	push @isbns, $input_isbn;
		402	}
		403	}
		404	$record_document->{$field} = \@isbns;
		405	}
		406	}
		407
380	# TODO: Perhaps should check if $records_document non empty, but really should never be the case	408	# TODO: Perhaps should check if $records_document non empty, but really should never be the case
381	$record->encoding('UTF-8');	409	$record->encoding('UTF-8');
382	my @warnings;	410	my @warnings;
Lines 470-475 sub get_marc_mapping_rules { Link Here
470	'control_fields' => {},	498	'control_fields' => {},
471	'data_fields' => {},	499	'data_fields' => {},
472	'sum' => [],	500	'sum' => [],
		501	'isbn' => [],
473	'defaults' => {}	502	'defaults' => {}
474	};	503	};
475		504
Lines 480-486 sub get_marc_mapping_rules { Link Here
480	if ($type eq 'sum') {	509	if ($type eq 'sum') {
481	push @{$rules->{sum}}, $name;	510	push @{$rules->{sum}}, $name;
482	}	511	}
483	elsif($type eq 'boolean') {	512	elsif ($type eq 'isbn') {
		513	push @{$rules->{isbn}}, $name;
		514	}
		515	elsif ($type eq 'boolean') {
484	# boolean gets special handling, if value doesn't exist for a field,	516	# boolean gets special handling, if value doesn't exist for a field,
485	# it is set to false	517	# it is set to false
486	$rules->{defaults}->{$name} = 'false';	518	$rules->{defaults}->{$name} = 'false';

Lines 115-126 subtest 'get_elasticsearch_mappings() tests' => sub { Link Here

(-)a/t/Koha/SearchEngine/Elasticsearch.t (-4 / +15 lines)
115		115
116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {	116	subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
117		117
118	plan tests => 30;	118	plan tests => 32;
119		119
120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');	120	t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
121		121
122	my @mappings = (	122	my @mappings = (
123	{	123	{
		124	name => 'isbn',
		125	type => 'isbn',
		126	facet => 0,
		127	suggestible => 0,
		128	sort => 0,
		129	marc_type => 'marc21',
		130	marc_field => '020a',
		131	},
		132	{
124	name => 'author',	133	name => 'author',
125	type => 'string',	134	type => 'string',
126	facet => 1,	135	facet => 1,
Lines 225-230 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
225	my $marc_record_1 = MARC::Record->new();	234	my $marc_record_1 = MARC::Record->new();
226	$marc_record_1->leader(' cam 22 a 4500');	235	$marc_record_1->leader(' cam 22 a 4500');
227	$marc_record_1->append_fields(	236	$marc_record_1->append_fields(
		237	MARC::Field->new('020', '', '', a => '1-56619-909-3'),
228	MARC::Field->new('100', '', '', a => 'Author 1'),	238	MARC::Field->new('100', '', '', a => 'Author 1'),
229	MARC::Field->new('110', '', '', a => 'Corp Author'),	239	MARC::Field->new('110', '', '', a => 'Corp Author'),
230	MARC::Field->new('210', '', '', a => 'Title 1'),	240	MARC::Field->new('210', '', '', a => 'Title 1'),
Lines 250-259 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
250	my $docs = $see->marc_records_to_documents($records);	260	my $docs = $see->marc_records_to_documents($records);
251		261
252	# First record:	262	# First record:
253
254	is(scalar @{$docs}, 2, 'Two records converted to documents');	263	is(scalar @{$docs}, 2, 'Two records converted to documents');
255		264
256	is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');	265	is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
257		266
258	is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');	267	is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
259	is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');	268	is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
Lines 325-330 subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' Link Here
325	'First document type_of_record_and_bib_level field should be set correctly'	334	'First document type_of_record_and_bib_level field should be set correctly'
326	);	335	);
327		336
		337	is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
		338	is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
		339
328	# Second record:	340	# Second record:
329		341
330	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');	342	is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
331	-