Attachment 76646 Details for Bug 20244 – Bug 20244: Improve Elasticsearch ISBN indexing

[patch] Bug 20244: Improve Elasticsearch ISBN indexing

Bug-20244-Improve-Elasticsearch-ISBN-indexing.patch (text/plain), 5.86 KB, created by Ere Maijala on 2018-07-03 11:16:31 UTC

(hide)

Description:

Filename:

MIME Type:

Creator: Ere Maijala

Created: 2018-07-03 11:16:31 UTC

Size: 5.86 KB

patch

obsolete

>From a03eb84e8baf5b892971488d817b13c8bf07f34b Mon Sep 17 00:00:00 2001
>From: Ere Maijala <ere.maijala@helsinki.fi>
>Date: Mon, 2 Jul 2018 15:55:10 +0300
>Subject: [PATCH] Bug 20244: Improve Elasticsearch ISBN indexing
>
>https://bugs.koha-community.org/show_bug.cgi?id=20244
>
>Test plan:
>
>1. Add a record with an ISBN-10 or ISBN-13 that can be converted to ISBN-10 (e.g. 1-56619-909-3).
>2. Verify that the record can be found by searching for "1-56619-909-3", "1566199093", "978-1-56619-909-4" and "9781566199094".
>---
> Koha/SearchEngine/Elasticsearch.pm  | 34 +++++++++++++++++++++++++++++++++-
> t/Koha/SearchEngine/Elasticsearch.t | 18 +++++++++++++++---
> 2 files changed, 48 insertions(+), 4 deletions(-)
>
>diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm
>index 32ba39b..81a26ee 100644
>--- a/Koha/SearchEngine/Elasticsearch.pm
>+++ b/Koha/SearchEngine/Elasticsearch.pm
>@@ -39,6 +39,7 @@ use Search::Elasticsearch;
> use MARC::File::XML;
> use MIME::Base64;
> use Encode qw(encode);
>+use Business::ISBN;
> 
> __PACKAGE__->mk_ro_accessors(qw( index ));
> __PACKAGE__->mk_accessors(qw( sort_fields ));
>@@ -377,6 +378,33 @@ sub marc_records_to_documents {
>                 $record_document->{$field} = sum0(grep { !ref($_) && m/\d+(\.\d+)?/} @{$record_document->{$field}});
>             }
>         }
>+        # Index all applicable ISBN forms (ISBN-10 and ISBN-13 with and without dashes)
>+        foreach my $field (@{$rules->{isbn}}) {
>+            if (defined $record_document->{$field}) {
>+                my @isbns = ();
>+                foreach my $input_isbn (@{$record_document->{$field}}) {
>+                    my $isbn = Business::ISBN->new($input_isbn);
>+                    if (defined $isbn && $isbn->is_valid) {
>+                        my $isbn13 = $isbn->as_isbn13->as_string;
>+                        push @isbns, $isbn13;
>+                        $isbn13 =~ s/\-//g;
>+                        push @isbns, $isbn13;
>+                        
>+                        my $isbn10 = $isbn->as_isbn10;
>+                        if ($isbn10) {
>+                            $isbn10 = $isbn10->as_string;
>+                            push @isbns, $isbn10;
>+                            $isbn10 =~ s/\-//g;
>+                            push @isbns, $isbn10;
>+                        }
>+                    } else {
>+                        push @isbns, $input_isbn;
>+                    }                
>+                }
>+                $record_document->{$field} = \@isbns;
>+            }
>+        }
>+
>         # TODO: Perhaps should check if $records_document non empty, but really should never be the case
>         $record->encoding('UTF-8');
>         my @warnings;
>@@ -470,6 +498,7 @@ sub get_marc_mapping_rules {
>         'control_fields' => {},
>         'data_fields' => {},
>         'sum' => [],
>+        'isbn' => [],
>         'defaults' => {}
>     };
> 
>@@ -480,7 +509,10 @@ sub get_marc_mapping_rules {
>         if ($type eq 'sum') {
>             push @{$rules->{sum}}, $name;
>         }
>-        elsif($type eq 'boolean') {
>+        elsif ($type eq 'isbn') {
>+            push @{$rules->{isbn}}, $name;
>+        }
>+        elsif ($type eq 'boolean') {
>             # boolean gets special handling, if value doesn't exist for a field,
>             # it is set to false
>             $rules->{defaults}->{$name} = 'false';
>diff --git a/t/Koha/SearchEngine/Elasticsearch.t b/t/Koha/SearchEngine/Elasticsearch.t
>index 55cf896..733e76f 100644
>--- a/t/Koha/SearchEngine/Elasticsearch.t
>+++ b/t/Koha/SearchEngine/Elasticsearch.t
>@@ -115,12 +115,21 @@ subtest 'get_elasticsearch_mappings() tests' => sub {
> 
> subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {
> 
>-    plan tests => 30;
>+    plan tests => 32;
> 
>     t::lib::Mocks::mock_preference('marcflavour', 'MARC21');
> 
>     my @mappings = (
>         {
>+            name => 'isbn',
>+            type => 'isbn',
>+            facet => 0,
>+            suggestible => 0,
>+            sort => 0,
>+            marc_type => 'marc21',
>+            marc_field => '020a',
>+        },
>+        {
>             name => 'author',
>             type => 'string',
>             facet => 1,
>@@ -225,6 +234,7 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     my $marc_record_1 = MARC::Record->new();
>     $marc_record_1->leader('     cam  22      a 4500');
>     $marc_record_1->append_fields(
>+        MARC::Field->new('020', '', '', a => '1-56619-909-3'),
>         MARC::Field->new('100', '', '', a => 'Author 1'),
>         MARC::Field->new('110', '', '', a => 'Corp Author'),
>         MARC::Field->new('210', '', '', a => 'Title 1'),
>@@ -250,10 +260,9 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>     my $docs = $see->marc_records_to_documents($records);
> 
>     # First record:
>-
>     is(scalar @{$docs}, 2, 'Two records converted to documents');
> 
>-    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');
>+    is($docs->[0][0], '1234567', 'First document biblionumber should be set as first element in document touple');    
> 
>     is(scalar @{$docs->[0][1]->{author}}, 2, 'First document author field should contain two values');
>     is_deeply($docs->[0][1]->{author}, ['Author 1', 'Corp Author'], 'First document author field should be set correctly');
>@@ -325,6 +334,9 @@ subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests'
>         'First document type_of_record_and_bib_level field should be set correctly'
>     );
> 
>+    is(scalar @{$docs->[0][1]->{isbn}}, 4, 'First document isbn field should contain four values');
>+    is_deeply($docs->[0][1]->{isbn}, ['978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093'], 'First document isbn field should be set correctly');
>+
>     # Second record:
> 
>     is(scalar @{$docs->[1][1]->{author}}, 1, 'Second document author field should contain one value');
>-- 
>2.7.4

Actions: View | Diff | Splinter Review

Attachments on bug 20244: 71919 | 73372 | 73373 | 74321 | 76646 | 76647 | 76648 | 76649 | 78727 | 78728 | 78729 | 79012 | 79013 | 79014 | 82160 | 82161 | 82162 | 82272 | 82273