From abf95d44d62cd9fcde88358e53746b9418d0b2b5 Mon Sep 17 00:00:00 2001 From: Janusz Kaczmarek Date: Fri, 10 Feb 2023 18:10:36 +0100 Subject: [PATCH] Bug 32916: [Bug 30280 follow-up] Problems in linking authorities to biblio fields (MARC 21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After applying the bug patch 30280: 1. Koha does not link with Zebra. 2. Koha does not properly link headings other than 6XX with ES. 3. Koha does not link subject headings when 6XX indicator 2 = "4" or in the case when in the auth record 008/11 = "z", but 040 $f is not defined, which is legal in terms of MARC 21 documentation (https://www.loc.gov/marc/authority/ad008.html): "A MARC code for the conventions used to formulate the heading may be contained in subfield $f (Subject heading/thesaurus conventions) in field 040 (Cataloging Source)." -- "may", not "should". A possible solution to this is to make ES emulate the Zebra indexing of auth 008/11 and a correction to C4::Heading::_search. Some minor corrections in other places had to be done. This solution requires a new ES field type and so also a modification of the database – an expansion of the ENUM type for search_field.type (ALTER TABLE search_field MODIFY COLUMN `type` ENUM('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','thesaurus') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine';) The new strict behaviour should be controlled (on/off) by a new preference (e.g. LinkerStrictAuthInfo) -- since not every library is using well formatted data. The semantics of this preference could be expanded in the future to take into account also 008/14-16 of an auth record for instance. Test plan: ========== 1. Have a clean master (or 22.11.0[0-2]) installation 2. Load provided data sample (bulkmarcimport.pl -d -a/b -file ...) 3. Reindex with Zebra and ES 4. Perform link_bibs_to_authorities.pl -v -t -l with Zebra, you should get: Linked headings (from most frequent to least): ------------------------------------------------------- Unlinked headings (from most frequent to least): ------------------------------------------------------- Feminism: 3 occurrences Author 01: 1 occurrences Author 01. Second work title: 1 occurrences feminism: 1 occurrences Person DBN: 1 occurrences Series entry: 1 occurrences Subject DBN without non-mandatory 040f: 1 occurrences Subject lcsh: 1 occurrences Subject with thesaurus not specified: 1 occurrences 5. Perform link_bibs_to_authorities.pl -v -t -l with ES, you should get: Linked headings (from most frequent to least): ------------------------------------------------------- Feminism: 2 occurrences feminism: 1 occurrences Person DBN: 1 occurrences Subject lcsh: 1 occurrences Unlinked headings (from most frequent to least): ------------------------------------------------------- Author 01: 1 occurrences Author 01. Second work title: 1 occurrences Feminism: 1 occurrences Series entry: 1 occurrences Subject DBN without non-mandatory 040f: 1 occurrences Subject with thesaurus not specified: 1 occurrences 6. Apply the patch (pay attention to the location of the authority-zebra-indexdefs.xsl file in you test environment). Add a system preference LinkerStrictAuthInfo = 1 and perform database modification (ALTER TABLE search_field MODIFY COLUMN `type` ENUM('','string','date','number','boolean','sum','isbn','stdno','year','callnumber','thesaurus') NOT NULL COMMENT 'what type of data this holds, relevant when storing it in the search engine';) 7. Full reindex with Zebra (koha-rebuild-zebra --full --force -a -b) and ES (koha-elasticsearch --rebuild -r -d -a -b) 8. Perform link_bibs_to_authorities.pl -v -t -l with Zebra and ES, you should get in both cases: Linked headings (from most frequent to least): ------------------------------------------------------- Feminism: 2 occurrences Author 01: 1 occurrences Author 01. Second work title: 1 occurrences feminism: 1 occurrences Person DBN: 1 occurrences Series entry: 1 occurrences Subject DBN without non-mandatory 040f: 1 occurrences Subject lcsh: 1 occurrences Subject with thesaurus not specified: 1 occurrences Unlinked headings (from most frequent to least): ------------------------------------------------------- Feminism: 1 occurrences 9. Perform link_bibs_to_authorities.pl (without -t) and control the results in Koha -- all the heading fields should be properly linked to the appropriate auth records. --- C4/Heading.pm | 54 +++++++++---------- C4/Heading/MARC21.pm | 4 +- Koha/SearchEngine/Elasticsearch.pm | 2 + .../Elasticsearch/QueryBuilder.pm | 1 - .../elasticsearch/field_config.yaml | 13 +++++ .../elasticsearch/index_config.yaml | 19 +++++++ .../searchengine/elasticsearch/mappings.yaml | 6 +-- .../authorities/authority-zebra-indexdefs.xsl | 3 ++ etc/zebradb/xsl/koha-indexdefs-to-zebra.xsl | 1 + .../searchengine/elasticsearch/mappings.tt | 5 ++ 10 files changed, 71 insertions(+), 37 deletions(-) diff --git a/C4/Heading.pm b/C4/Heading.pm index 7541a56205..a143509ad1 100644 --- a/C4/Heading.pm +++ b/C4/Heading.pm @@ -194,8 +194,7 @@ sub _search { my $self = shift; my $index = shift || undef; my $skipmetadata = shift || undef; - my $ind2 = $self->{field}->{_ind2}; - my $subject_heading_thesaurus = ''; + my $ind2 = $self->{field}->indicator('2'); my @marclist; my @and_or; my @excluding = []; @@ -209,36 +208,11 @@ sub _search { push @value, $self->{'search_form'}; } - if ($self->{'thesaurus'}) { - if ($ind2 eq '0') { - $subject_heading_thesaurus = 'a'; - } elsif ($ind2 eq '1') { - $subject_heading_thesaurus = 'b'; - } elsif ($ind2 eq '2') { - $subject_heading_thesaurus = 'c'; - } elsif ($ind2 eq '3') { - $subject_heading_thesaurus = 'd'; - } elsif ($ind2 eq '4') { - $subject_heading_thesaurus = 'n'; - } elsif ($ind2 eq '5') { - $subject_heading_thesaurus = 'k'; - } elsif ($ind2 eq '6') { - $subject_heading_thesaurus = 'v'; - } else { - $subject_heading_thesaurus = 'z'; - } + if (C4::Context->preference("LinkerStrictAuthInfo") && $self->{'thesaurus'}) { push @marclist, 'thesaurus'; push @and_or, 'and'; push @excluding, ''; push @operator, 'is'; - push @value, $subject_heading_thesaurus; - } - - if ($ind2 eq '7') { - push @marclist, 'thesaurus-conventions'; - push @and_or, 'and'; - push @excluding, ''; - push @operator, 'is'; push @value, $self->{'thesaurus'}; } @@ -258,7 +232,29 @@ sub _search { \@value, $self->{'auth_type'}, 'AuthidAsc' ); - return $searcher->search_auth_compat( $search_query, 0, 20, $skipmetadata ); + my ( $authresults, $total ) = $searcher->search_auth_compat( $search_query, 0, 20, $skipmetadata ); + if (C4::Context->preference("LinkerStrictAuthInfo") && $self->{'thesaurus'} && ! $total && $ind2 eq '7') { + # For the case when auth 008/11 was 'z', but thesaurus was not declared in 040 $f + # (which is legal in MARC 21) we try to search again with 'other' (that stands + # for 008/11 = 'z') instead of biblio 6XX $7 (i.e. $self->{'thesaurus'}) + my $thesaurus_pos; + for my $i (0..$#marclist) { + if ($marclist[$i] eq 'thesaurus') { + $thesaurus_pos = $i; + last; + } + } + if ($thesaurus_pos) { + $value[$thesaurus_pos] = 'notdefined'; + $search_query = $builder->build_authorities_query_compat( + \@marclist, \@and_or, \@excluding, \@operator, + \@value, $self->{'auth_type'}, + 'AuthidAsc' + ); + ( $authresults, $total ) = $searcher->search_auth_compat( $search_query, 0, 20, $skipmetadata ); + } + } + return ( $authresults, $total ); } =head1 INTERNAL FUNCTIONS diff --git a/C4/Heading/MARC21.pm b/C4/Heading/MARC21.pm index 9ae5827541..e25d920c26 100644 --- a/C4/Heading/MARC21.pm +++ b/C4/Heading/MARC21.pm @@ -339,7 +339,7 @@ sub parse_heading { my $thesaurus = $tag =~ m/6../ ? _get_subject_thesaurus($field) - : "lcsh"; # use 'lcsh' for names, UT, etc. + : undef; # defining thesaurus makes sence only for 6XX my $search_heading = _get_search_heading( $field, $field_info->{'subfields'} ); my $display_heading = @@ -383,7 +383,7 @@ sub _get_subject_thesaurus { } elsif ( $ind2 eq '7' ) { my $sf2 = $field->subfield('2'); - $thesaurus = $sf2 if defined($sf2); + $thesaurus = $sf2 ? $sf2 : 'notdefined'; } return $thesaurus; diff --git a/Koha/SearchEngine/Elasticsearch.pm b/Koha/SearchEngine/Elasticsearch.pm index 1fe6fb1c6f..63d8fd4673 100644 --- a/Koha/SearchEngine/Elasticsearch.pm +++ b/Koha/SearchEngine/Elasticsearch.pm @@ -212,6 +212,8 @@ sub get_elasticsearch_mappings { $es_type = 'year'; } elsif ($type eq 'callnumber') { $es_type = 'cn_sort'; + } elsif ($type eq 'thesaurus') { + $es_type = 'thesaurus'; } if ($search) { diff --git a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm index 53042b4303..966f65fa4e 100644 --- a/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm +++ b/Koha/SearchEngine/Elasticsearch/QueryBuilder.pm @@ -559,7 +559,6 @@ our $koha_to_index_name = { 'match-heading' => 'match-heading', 'see-from' => 'match-heading-see-from', thesaurus => 'subject-heading-thesaurus', - 'thesaurus-conventions' => 'subject-heading-thesaurus-conventions', any => '', all => '' }; diff --git a/admin/searchengine/elasticsearch/field_config.yaml b/admin/searchengine/elasticsearch/field_config.yaml index cf90482aa2..ea5545d102 100644 --- a/admin/searchengine/elasticsearch/field_config.yaml +++ b/admin/searchengine/elasticsearch/field_config.yaml @@ -38,6 +38,19 @@ search: search_analyzer: analyzer_phrase raw: type: keyword + thesaurus: + type: text + analyzer: analyzer_thesaurus + search_analyzer: analyzer_thesaurus + fields: + raw: + type: text + analyzer: analyzer_thesaurus + search_analyzer: analyzer_thesaurus + ci_raw: + type: text + analyzer: analyzer_thesaurus + search_analyzer: analyzer_thesaurus default: type: text analyzer: analyzer_standard diff --git a/admin/searchengine/elasticsearch/index_config.yaml b/admin/searchengine/elasticsearch/index_config.yaml index b53578fef0..a1f3453e81 100644 --- a/admin/searchengine/elasticsearch/index_config.yaml +++ b/admin/searchengine/elasticsearch/index_config.yaml @@ -20,6 +20,10 @@ index: - icu_folding char_filter: - punctuation + analyzer_thesaurus: + tokenizer: keyword + filter: + - thesaurus normalizer: icu_folding_normalizer: type: custom @@ -41,6 +45,21 @@ index: type: pattern_replace pattern: '\s*(? lcsh + - b => lcac + - c => mesh + - d => nal + - k => cash + - n => notapplicable + - r => aat + - s => sears + - v => rvm + - z => notdefined + - "| => notspecified" index.mapping.total_fields.limit: 10000 index.max_result_window: 1000000 index.number_of_replicas: 1 diff --git a/admin/searchengine/elasticsearch/mappings.yaml b/admin/searchengine/elasticsearch/mappings.yaml index e892f9126c..a970323c97 100644 --- a/admin/searchengine/elasticsearch/mappings.yaml +++ b/admin/searchengine/elasticsearch/mappings.yaml @@ -813,16 +813,12 @@ authorities: marc_type: marc21 sort: ~ suggestible: '' - type: '' - Subject-heading-thesaurus-conventions: - label: Subject-heading-thesaurus-conventions - mappings: - facet: '' marc_field: 040f marc_type: marc21 sort: ~ suggestible: '' - type: '' + type: thesaurus authtype: label: authtype mappings: diff --git a/etc/zebradb/marc_defs/marc21/authorities/authority-zebra-indexdefs.xsl b/etc/zebradb/marc_defs/marc21/authorities/authority-zebra-indexdefs.xsl index 0723caf2b4..34eb698919 100644 --- a/etc/zebradb/marc_defs/marc21/authorities/authority-zebra-indexdefs.xsl +++ b/etc/zebradb/marc_defs/marc21/authorities/authority-zebra-indexdefs.xsl @@ -2441,6 +2441,9 @@ definition file (probably something like {biblio,authority}-koha-indexdefs.xml) rvm + + notspecified + diff --git a/etc/zebradb/xsl/koha-indexdefs-to-zebra.xsl b/etc/zebradb/xsl/koha-indexdefs-to-zebra.xsl index e12f73b192..d69ed7a921 100644 --- a/etc/zebradb/xsl/koha-indexdefs-to-zebra.xsl +++ b/etc/zebradb/xsl/koha-indexdefs-to-zebra.xsl @@ -158,6 +158,7 @@ definition file (probably something like {biblio,authority}-koha-indexdefs.xml) aat sears rvm + notspecified diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt index b0ec83750f..1fa0874f48 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/searchengine/elasticsearch/mappings.tt @@ -227,6 +227,11 @@ a.add, a.delete { [% ELSE %] [% END %] + [% IF search_field.type == "thesaurus" %] + + [% ELSE %] + + [% END %] -- 2.30.2