From 8e143143d7562a633c891c1f62128fa5c6336b6f Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Thu, 2 Feb 2017 13:36:38 +0100 Subject: [PATCH] Bug 14957: Merge rules system for merging of MARC records Add a rule based system for merging MARC records to for example prevent field data from being overwritten. To test: 1. Apply this patch. 2. Log in to staff client. 3. Enable new syspref MARCMergeRules. 4. Click the new link "MARC merge rules" in the "Catalog" section of the Koha administration page. 5. Create a new rule: Module: source, Filter: *, Tag: 245a, Preset: Protect. 6. Clicking "Edit" should allow you to edit corresponding rule. 7. Clicking "Delete" should remove corresponding rule after confirmation. 8. Selecting one or more rules followed by clicking "Delete selected" should remove all selected rules after confirmation. 9. Try creating a rule with tag set to "**", the other options does not matter. Verify that saving this rule produces an error message complaining about invalid tag regular expression. 10. Try creating a rule with tag set to "008" (or other control field) and set Appended: Append and Removed: Skip, the other options does not matter. Verify that saving this rule produces an error message complaining about invalid combination of actions for control field. 11. With the 245a rule in step 5 in place, edit a bibliographic record, change 245a (which should be Title for MARC21) and save. 12. Verify that the changes has not been saved. 13. Create a new rule: Module: source, Filter: intranet, Tag: 245a, Preset: Overwrite. 14. Repeat step 12, and verify that the changes has now been saved. 15. Run tests in t/db_dependent/Biblio/MarcMergeRules.t and very that all tests pass. Sponsored-by: Halland County Library Sponsored-by: Catalyst IT Sponsored-by: Gothenbug University Library --- C4/Biblio.pm | 95 ++- C4/ImportBatch.pm | 2 +- Koha/Exceptions/MarcMergeRule.pm | 55 ++ Koha/MarcMergeRule.pm | 72 ++ Koha/MarcMergeRules.pm | 312 +++++++++ Koha/MarcMergeRulesModule.pm | 40 ++ Koha/MarcMergeRulesModules.pm | 31 + admin/marc-merge-rules.pl | 157 +++++ cataloguing/addbiblio.pl | 12 +- .../atomicupdate/bug_14957-marc-merge-rules.sql | 30 + .../atomicupdate/bug_14957-merge-rules-syspref.sql | 1 + .../intranet-tmpl/prog/en/includes/admin-menu.inc | 1 + .../prog/en/modules/admin/admin-home.tt | 2 + .../prog/en/modules/admin/marc-merge-rules.tt | 395 +++++++++++ .../en/modules/admin/preferences/cataloguing.pref | 7 + .../prog/en/modules/cataloguing/addbiblio.tt | 1 + misc/migration_tools/bulkmarcimport.pl | 2 +- misc/migration_tools/import_lexile.pl | 3 +- t/db_dependent/Biblio/MarcMergeRules.t | 721 +++++++++++++++++++++ tools/batch_record_modification.pl | 13 +- 20 files changed, 1936 insertions(+), 16 deletions(-) create mode 100644 Koha/Exceptions/MarcMergeRule.pm create mode 100644 Koha/MarcMergeRule.pm create mode 100644 Koha/MarcMergeRules.pm create mode 100644 Koha/MarcMergeRulesModule.pm create mode 100644 Koha/MarcMergeRulesModules.pm create mode 100755 admin/marc-merge-rules.pl create mode 100644 installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql create mode 100644 installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt create mode 100755 t/db_dependent/Biblio/MarcMergeRules.t diff --git a/C4/Biblio.pm b/C4/Biblio.pm index 50c7c4cc10..d1dd0bb3b6 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -64,6 +64,7 @@ BEGIN { DelBiblio BiblioAutoLink LinkBibHeadingsToAuthorities + ApplyMarcMergeRules TransformMarcToKoha TransformHtmlToMarc TransformHtmlToXml @@ -107,10 +108,10 @@ use Koha::Holds; use Koha::ItemTypes; use Koha::SearchEngine; use Koha::Libraries; +use Koha::MarcMergeRules; use vars qw($debug $cgi_debug); - =head1 NAME C4::Biblio - cataloging management functions @@ -240,7 +241,7 @@ sub AddBiblio { =head2 ModBiblio - ModBiblio( $record,$biblionumber,$frameworkcode); + ModBiblio($record, $biblionumber, $frameworkcode, $options); Replace an existing bib record identified by C<$biblionumber> with one supplied by the MARC::Record object C<$record>. The embedded @@ -261,7 +262,9 @@ Returns 1 on success 0 on failure =cut sub ModBiblio { - my ( $record, $biblionumber, $frameworkcode ) = @_; + my ( $record, $biblionumber, $frameworkcode, $options ) = @_; + $options //= {}; + if (!$record) { carp 'No record passed to ModBiblio'; return 0; @@ -293,6 +296,16 @@ sub ModBiblio { _strip_item_fields($record, $frameworkcode); + # apply merge rules + if (C4::Context->preference('MARCMergeRules') && $biblionumber && defined $options && exists $options->{'context'}) { + $record = ApplyMarcMergeRules({ + biblionumber => $biblionumber, + record => $record, + context => $options->{'context'}, + } + ); + } + # update biblionumber and biblioitemnumber in MARC # FIXME - this is assuming a 1 to 1 relationship between # biblios and biblioitems @@ -303,13 +316,13 @@ sub ModBiblio { _koha_marc_update_bib_ids( $record, $frameworkcode, $biblionumber, $biblioitemnumber ); # load the koha-table data object - my $oldbiblio = TransformMarcToKoha( $record, $frameworkcode ); + my $oldbiblio = TransformMarcToKoha( $record, $frameworkcode, undef, $biblionumber); # update MARC subfield that stores biblioitems.cn_sort _koha_marc_update_biblioitem_cn_sort( $record, $oldbiblio, $frameworkcode ); # update the MARC record (that now contains biblio and items) with the new record data - &ModBiblioMarc( $record, $biblionumber, $frameworkcode ); + ModBiblioMarc( $record, $biblionumber, $frameworkcode ); # modify the other koha tables _koha_modify_biblio( $dbh, $oldbiblio, $frameworkcode ); @@ -2525,7 +2538,7 @@ sub TransformHtmlToMarc { =head2 TransformMarcToKoha - $result = TransformMarcToKoha( $record, undef, $limit ) + $result = TransformMarcToKoha($record, undef, $biblionumber, $option) Extract data from a MARC bib record into a hashref representing Koha biblio, biblioitems, and items fields. @@ -2536,7 +2549,7 @@ hash_ref. =cut sub TransformMarcToKoha { - my ( $record, $frameworkcode, $limit_table ) = @_; + my ( $record, $frameworkcode, $limit_table, $biblionumber ) = @_; # FIXME Parameter $frameworkcode is obsolete and will be removed $limit_table //= q{}; @@ -3223,7 +3236,7 @@ sub _koha_delete_biblio_metadata { =head2 ModBiblioMarc - &ModBiblioMarc($newrec,$biblionumber,$frameworkcode); + &ModBiblioMarc($newrec, $biblionumber, $frameworkcode, $options); Add MARC XML data for a biblio to koha @@ -3550,8 +3563,72 @@ sub RemoveAllNsb { return $record; } -1; +=head2 ApplyMarcMergeRules + + my $record = ApplyMarcMergeRules($params) + +Applies marc merge rules to a record. + +C<$params> is expected to be a hashref with below keys defined. + +=over 4 + +=item C +biblionumber of old record + +=item C +Incoming record that will be merged with old record + +=item C +hashref containing at least one module from marc_merge_rules_modules table +with filter value on the form {module => filter, ...}. Three predefined +context modules exists: + + * source + * category + * borrower + +=back + +Returns: +=over 4 + +=item C<$record> + +Merged MARC record based with merge rules for C applied. If no old +record for C can be found, C is returned unchanged. +Default action when no matching context is found to return C unchanged. +If no rules are found for a certain field tag the default is to overwrite with +fields with this field tag from C. + +=back + +=cut + +sub ApplyMarcMergeRules { + my ($params) = @_; + my $biblionumber = $params->{biblionumber}; + my $incoming_record = $params->{record}; + + if (!$biblionumber) { + carp 'ApplyMarcMergeRules called on undefined biblionumber'; + return; + } + if (!$incoming_record) { + carp 'ApplyMarcMergeRules called on undefined record'; + return; + } + my $old_record = GetMarcBiblio({ biblionumber => $biblionumber }); + + # Skip merge rules if called with no context + if ($old_record && defined $params->{context}) { + return Koha::MarcMergeRules->merge_records($old_record, $incoming_record, $params->{context}); + } + return $incoming_record; +} + +1; __END__ diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index 99bc428701..3804ba0ed0 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -667,7 +667,7 @@ sub BatchCommitRecords { } $oldxml = $old_marc->as_xml($marc_type); - ModBiblio($marc_record, $recordid, $oldbiblio->frameworkcode); + ModBiblio($marc_record, $recordid, $oldbiblio->frameworkcode, {context => {source => 'batchimport'}}); $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; if ($item_result eq 'create_new' || $item_result eq 'replace') { diff --git a/Koha/Exceptions/MarcMergeRule.pm b/Koha/Exceptions/MarcMergeRule.pm new file mode 100644 index 0000000000..a3cd708c45 --- /dev/null +++ b/Koha/Exceptions/MarcMergeRule.pm @@ -0,0 +1,55 @@ +package Koha::Exceptions::MarcMergeRule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use Exception::Class ( + + 'Koha::Exceptions::MarcMergeRule' => { + description => 'Something went wrong!', + }, + 'Koha::Exceptions::MarcMergeRule::InvalidTagRegExp' => { + isa => 'Koha::Exceptions::MarcMergeRule', + description => 'Invalid regular expression for tag' + }, + 'Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions' => { + isa => 'Koha::Exceptions::MarcMergeRule', + description => 'Invalid control field actions' + } +); + +=head1 NAME + +Koha::Exceptions::MarcMergeRule - Base class for MarcMergeRule exceptions + +=head1 Exceptions + +=head2 Koha::Exceptions::MarcMergeRule + +Generic MarcMergeRule exception + +=head2 Koha::Exceptions::MarcMergeRule::InvalidTagRegExp + +Exception for rule validation when rule tag is an invalid regular expression + +=head2 Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions + +Exception for rule validation for control field rules with invalid combination of actions + +=cut + +1; diff --git a/Koha/MarcMergeRule.pm b/Koha/MarcMergeRule.pm new file mode 100644 index 0000000000..b228b90146 --- /dev/null +++ b/Koha/MarcMergeRule.pm @@ -0,0 +1,72 @@ +package Koha::MarcMergeRule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Koha::Exceptions::MarcMergeRule; +use Try::Tiny; +use Scalar::Util qw(looks_like_number); + +use parent qw(Koha::Object); + +my $cache = Koha::Caches->get_instance(); + +=head1 NAME + +Koha::MarcMergeRule - Koha SearchField Object class + +=cut + +sub set { + my $self = shift @_; + my ($rule_data) = @_; + + if(exists $rule_data->{tag}) { + if ($rule_data->{tag} ne '*') { + eval { qr/$rule_data->{tag}/ }; + if ($@) { + Koha::Exceptions::MarcMergeRule::InvalidTagRegExp->throw( + "Invalid tag regular expression" + ); + } + } + # TODO: Regexp or '*' that match controlfield not currently detected + if ( + looks_like_number($rule_data->{tag}) && + $rule_data->{tag} < 10 && + $rule_data->{append} && + !$rule_data->{remove} + ) { + Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions->throw( + "Combination of allow append and skip remove not permitted for control fields" + ); + } + } + + $self->SUPER::set(@_); +} + +sub store { + my $self = shift @_; + $cache->clear_from_cache('marc_merge_rules'); + $self->SUPER::store(@_); +} + +sub _type { + return 'MarcMergeRule'; +} + +1; diff --git a/Koha/MarcMergeRules.pm b/Koha/MarcMergeRules.pm new file mode 100644 index 0000000000..229c242da2 --- /dev/null +++ b/Koha/MarcMergeRules.pm @@ -0,0 +1,312 @@ +package Koha::MarcMergeRules; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use List::Util qw(first); +use Koha::MarcMergeRule; +use Carp; + +use parent qw(Koha::Objects); + +sub operations { + return ('add', 'append', 'remove', 'delete'); +} + +my $cache = Koha::Caches->get_instance(); + +=head1 NAME + +Koha::MarcMergeRules - Koha MarcMergeRules Object set class + +=head1 API + +=head2 Class Methods + +=head3 context_rules + + my $rules = Koha::MarcMergeRules->context_rules($context); + +Gets all MARC merge rules for the supplied C<$context> (hashref with { module => filter, ... } values). + +=cut + +sub context_rules { + my ($self, $context) = @_; + + return unless %{$context}; + + my $rules = $cache->get_from_cache('marc_merge_rules', { unsafe => 1 }); + + if (!$rules) { + $rules = {}; + my @rules_rows = $self->_resultset()->search( + undef, + { + prefetch => 'module', + order_by => { -desc => [qw/module.specificity me.id/] } + } + ); + foreach my $rule_row (@rules_rows) { + my %rule = $rule_row->get_columns(); + my $operations = {}; + + foreach my $operation ($self->operations) { + $operations->{$operation} = { allow => $rule{$operation}, rule => $rule{id} }; + } + + # TODO: Remove unless check and validate on saving rules? + if ($rule{tag} eq '*') { + unless (exists $rules->{$rule{module}}->{$rule{filter}}->{'*'}) { + $rules->{$rule{module}}->{$rule{filter}}->{'*'} = $operations; + } + } + elsif ($rule{tag} =~ /^(\d{3})$/) { + unless (exists $rules->{$rule{module}}->{$rule{filter}}->{tags}->{$rule{tag}}) { + $rules->{$rule{module}}->{$rule{filter}}->{tags}->{$rule{tag}} = $operations; + } + } + else { + my $regexps = ($rules->{$rule{module}}->{$rule{filter}}->{regexps} //= []); + push @{$regexps}, [$rule{tag}, $operations]; + } + } + $cache->set_in_cache('marc_merge_rules', $rules); + } + + my $context_rules = undef; + foreach my $module_name (keys %{$context}) { + if ( + exists $rules->{$module_name} && + exists $rules->{$module_name}->{$context->{$module_name}} + ) { + $context_rules = $rules->{$module_name}->{$context->{$module_name}}; + last; + } + } + if (!$context_rules) { + # No perms matching specific context conditions found, try wildcard value for each active context + foreach my $module_name (keys %{$context}) { + if (exists $rules->{$module_name}->{'*'}) { + $context_rules = $rules->{$module_name}->{'*'}; + last; + } + } + } + return $context_rules; +} + +=head3 merge_records + + my $merged_record = Koha::MarcMergeRules->merge_records($old_record, $incoming_record, $context); + +Merge C<$old_record> with C<$incoming_record> applying merge rules for C<$context>. +Returns merged record C<$merged_record>. C<$old_record>, C<$incoming_record> and +C<$merged_record> are all MARC::Record objects. + +=cut + +sub merge_records { + my ($self, $old_record, $incoming_record, $context) = @_; + + my $rules = $self->context_rules($context); + + # Default when no rules found is to overwrite with incoming record + return $incoming_record unless $rules; + + my $fields_by_tag = sub { + my ($record) = @_; + my $fields = {}; + foreach my $field ($record->fields()) { + $fields->{$field->tag()} //= []; + push @{$fields->{$field->tag()}}, $field; + } + return $fields; + }; + + my $hash_field_data = sub { + my ($field) = @_; + my $indicators = join("\x1E", map { $field->indicator($_) } (1, 2)); + return $indicators . "\x1E" . join("\x1E", sort map { join "\x1E", @{$_} } $field->subfields()); + }; + + my $diff_by_key = sub { + my ($a, $b) = @_; + my @removed; + my @intersecting; + my @added; + my %keys_index = map { $_ => undef } (keys %{$a}, keys %{$b}); + foreach my $key (keys %keys_index) { + if ($a->{$key} && $b->{$key}) { + push @intersecting, $a->{$key}; + } + elsif ($a->{$key}) { + push @removed, $a->{$key}; + } + else { + push @added, $b->{$key}; + } + } + return (\@removed, \@intersecting, \@added); + }; + + my $tag_rules = $rules->{tags} // {}; + my $default_rule = $rules->{'*'} // { + add => { allow => 1, 'rule' => 0}, + append => { allow => 1, 'rule' => 0}, + delete => { allow => 1, 'rule' => 0}, + remove => { allow => 1, 'rule' => 0}, + }; + + # Precompile regexps + my @regexp_rules = map { { regexp => qr/^$_->[0]$/, actions => $_->[1] } } @{$rules->{regexps} // []}; + + my $get_matching_field_rule = sub { + my ($tag) = @_; + # Exact match takes precedence, then regexp, then wildcard/defaults + return $tag_rules->{$tag} // + %{(first { $tag =~ $_->{regexp} } @regexp_rules) // {}}{actions} // + $default_rule; + }; + + my %merged_record_fields; + + my $current_fields = $fields_by_tag->($old_record); + my $incoming_fields = $fields_by_tag->($incoming_record); + + # First we get all new incoming fields + my @new_field_tags = grep { !(exists $current_fields->{$_}) } keys %{$incoming_fields}; + foreach my $tag (@new_field_tags) { + my $rule = $get_matching_field_rule->($tag); + if ($rule->{add}->{allow}) { + $merged_record_fields{$tag} //= []; + push @{$merged_record_fields{$tag}}, @{$incoming_fields->{$tag}}; + } + } + + # Then we get all fields no longer present in incoming fields + my @deleted_field_tags = grep { !(exists $incoming_fields->{$_}) } keys %{$current_fields}; + foreach my $tag (@deleted_field_tags) { + my $rule = $get_matching_field_rule->($tag); + if (!$rule->{delete}->{allow}) { + $merged_record_fields{$tag} //= []; + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + } + + # Then we get the intersection of fields, present both in + # current and incoming record (possibly to be overwritten) + my @common_field_tags = grep { exists $incoming_fields->{$_} } keys %{$current_fields}; + foreach my $tag (@common_field_tags) { + my $rule = $get_matching_field_rule->($tag); + + # Special handling for control fields + if ($tag < 10) { + if ( + $rule->{append}->{allow} && + !$rule->{remove}->{allow} + ) { + # This should be highly unlikely since we have input validation to protect against this case + carp "Allowing \"append\" and skipping \"remove\" is not permitted for control fields, falling back to skipping both \"append\" and \"remove\""; + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + elsif ($rule->{append}->{allow}) { + push @{$merged_record_fields{$tag}}, @{$incoming_fields->{$tag}}; + } + else { + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + } + else { + # Compute intersection and diff using field data + my $sort_weight = 0; + my %current_fields_by_data = map { $hash_field_data->($_) => [$sort_weight++, $_] } @{$current_fields->{$tag}}; + + # Always put incoming fields after current fields + my %incoming_fields_by_data = map { $hash_field_data->($_) => [$sort_weight++, $_] } @{$incoming_fields->{$tag}}; + + my ($current_fields_only, $common_fields, $incoming_fields_only) = $diff_by_key->(\%current_fields_by_data, \%incoming_fields_by_data); + + my @merged_fields; + + # First add common fields (intersection) + # Unchanged + if (@{$common_fields}) { + push @merged_fields, @{$common_fields}; + } + # Removed + if (@{$current_fields_only}) { + if (!$rule->{remove}->{allow}) { + push @merged_fields, @{$current_fields_only}; + } + } + # Appended + if (@{$incoming_fields_only}) { + if ($rule->{append}->{allow}) { + push @merged_fields, @{$incoming_fields_only}; + } + } + $merged_record_fields{$tag} //= []; + + # Sort ascending according to weight (original order) + push @{$merged_record_fields{$tag}}, map { $_->[1] } sort { $a->[0] <=> $b->[0] } @merged_fields; + } + } + + my $merged_record = MARC::Record->new(); + + # Leader is always overwritten, or kept??? + $merged_record->leader($incoming_record->leader()); + + if (%merged_record_fields) { + foreach my $tag (sort keys %merged_record_fields) { + $merged_record->append_fields(@{$merged_record_fields{$tag}}); + } + } + return $merged_record; +} + +sub _clear_caches { + $cache->clear_from_cache('marc_merge_rules'); +} + +sub find_or_create { + my $self = shift @_; + $self->_clear_caches(); + $self->SUPER::find_or_create(@_); +} + +sub update { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::update(@_); +} + +sub delete { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::delete(@_); +} + +sub _type { + return 'MarcMergeRule'; +} + +sub object_class { + return 'Koha::MarcMergeRule'; +} + +1; diff --git a/Koha/MarcMergeRulesModule.pm b/Koha/MarcMergeRulesModule.pm new file mode 100644 index 0000000000..9695759dc9 --- /dev/null +++ b/Koha/MarcMergeRulesModule.pm @@ -0,0 +1,40 @@ +package Koha::MarcMergeRulesModule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use parent qw(Koha::Object); + +=head1 NAME + +Koha::MarcMergeRulesModule - Koha SearchField Object class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub _type { + return 'MarcMergeRulesModule'; +} + +1; diff --git a/Koha/MarcMergeRulesModules.pm b/Koha/MarcMergeRulesModules.pm new file mode 100644 index 0000000000..6c1b163699 --- /dev/null +++ b/Koha/MarcMergeRulesModules.pm @@ -0,0 +1,31 @@ +package Koha::MarcMergeRulesModules; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Koha::MarcMergeRulesModule; + +use parent qw(Koha::Objects); + +sub _type { + return 'MarcMergeRulesModule'; +} + +sub object_class { + return 'Koha::MarcMergeRulesModule'; +} + +1; diff --git a/admin/marc-merge-rules.pl b/admin/marc-merge-rules.pl new file mode 100755 index 0000000000..3e45ba08f2 --- /dev/null +++ b/admin/marc-merge-rules.pl @@ -0,0 +1,157 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use strict; +use warnings; + +# standard or CPAN modules used +use CGI qw ( -utf8 ); +use CGI::Cookie; +use MARC::File::USMARC; +use Try::Tiny; + +# Koha modules used +use C4::Context; +use C4::Koha; +use C4::Auth; +use C4::AuthoritiesMarc; +use C4::Output; +use C4::Biblio; +use C4::ImportBatch; +use C4::Matcher; +use C4::BackgroundJob; +use C4::Labels::Batch; +use Koha::MarcMergeRules; +use Koha::MarcMergeRule; +use Koha::MarcMergeRulesModules; + +my $script_name = "/cgi-bin/koha/admin/marc-merge-rules.pl"; + +my $input = new CGI; +my $op = $input->param('op') || ''; +my $errors = []; + +my $rule_from_cgi = sub { + my ($cgi) = @_; + + my %rule = map { $_ => scalar $cgi->param($_) } ( + 'tag', + 'module', + 'filter', + 'add', + 'append', + 'remove', + 'delete' + ); + + my $id = $cgi->param('id'); + if ($id) { + $rule{id} = $id; + } + + return \%rule; +}; + +my ($template, $loggedinuser, $cookie) = get_template_and_user( + { + template_name => "admin/marc-merge-rules.tt", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => { parameters => 'parameters_remaining_permissions' }, + debug => 1, + } +); + +my %cookies = parse CGI::Cookie($cookie); +our $sessionID = $cookies{'CGISESSID'}->value; + +my $get_rules = sub { + # TODO: order? + return [map { { $_->get_columns() } } Koha::MarcMergeRules->_resultset->all]; +}; +my $rules; + +if ($op eq 'remove' || $op eq 'doremove') { + my @remove_ids = $input->multi_param('batchremove'); + push @remove_ids, scalar $input->param('id') if $input->param('id'); + if ($op eq 'remove') { + $template->{VARS}->{removeConfirm} = 1; + my %remove_ids = map { $_ => undef } @remove_ids; + $rules = $get_rules->(); + for my $rule (@{$rules}) { + $rule->{'removemarked'} = 1 if exists $remove_ids{$rule->{id}}; + } + } + elsif ($op eq 'doremove') { + my @remove_ids = $input->multi_param('batchremove'); + push @remove_ids, scalar $input->param('id') if $input->param('id'); + Koha::MarcMergeRules->search({ id => { in => \@remove_ids } })->delete(); + $rules = $get_rules->(); + } +} +elsif ($op eq 'edit') { + $template->{VARS}->{edit} = 1; + my $id = $input->param('id'); + $rules = $get_rules->(); + for my $rule(@{$rules}) { + if ($rule->{id} == $id) { + $rule->{'edit'} = 1; + last; + } + } +} +elsif ($op eq 'doedit' || $op eq 'add') { + my $rule_data = $rule_from_cgi->($input); + if (!@{$errors}) { + my $rule = Koha::MarcMergeRules->find_or_create($rule_data); + try { + $rule->set($rule_data); + $rule->store(); + } + catch { + die $_ unless blessed $_ && $_->can('rethrow'); + + if ($_->isa('Koha::Exceptions::MarcMergeRule::InvalidTagRegExp')) { + push @{$errors}, { + type => 'error', + code => 'invalid_tag_regexp', + tag => $rule_data->{tag}, + }; + } + elsif ($_->isa('Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions')) { + push @{$errors}, { + type => 'error', + code => 'invalid_control_field_actions', + tag => $rule_data->{tag}, + }; + } + else { + $_->rethrow; + } + }; + $rules = $get_rules->(); + } +} +else { + $rules = $get_rules->(); +} + +my $modules = Koha::MarcMergeRulesModules->as_list; +$template->param( rules => $rules, modules => $modules, messages => $errors ); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/cataloguing/addbiblio.pl b/cataloguing/addbiblio.pl index 901892936b..7c5bb18f50 100755 --- a/cataloguing/addbiblio.pl +++ b/cataloguing/addbiblio.pl @@ -41,6 +41,7 @@ use Koha::ItemTypes; use Koha::Libraries; use Koha::BiblioFrameworks; +use Koha::Patrons; use Date::Calc qw(Today); use MARC::File::USMARC; @@ -832,7 +833,15 @@ if ( $op eq "addbiblio" ) { if ( !$duplicatebiblionumber or $confirm_not_duplicate ) { my $oldbibitemnum; if ( $is_a_modif ) { - ModBiblio( $record, $biblionumber, $frameworkcode ); + my $member = Koha::Patrons->find($loggedinuser); + ModBiblio( $record, $biblionumber, $frameworkcode, { + context => { + source => $z3950 ? 'z39.50' : 'intranet', + category => $member->{'category_type'}, + borrower => $loggedinuser + } + } + ); } else { ( $biblionumber, $oldbibitemnum ) = AddBiblio( $record, $frameworkcode ); @@ -925,6 +934,7 @@ elsif ( $op eq "delete" ) { $template->param( biblionumberdata => $biblionumber, op => $op, + z3950 => $z3950 ); if ( $op eq "duplicate" ) { $biblionumber = ""; diff --git a/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql b/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql new file mode 100644 index 0000000000..b2d99ee3e7 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS `marc_merge_rules`; +DROP TABLE IF EXISTS `marc_merge_rules_modules`; + +CREATE TABLE `marc_merge_rules_modules` ( + `name` varchar(255) NOT NULL, + `description` varchar(255), + `specificity` int(11) NOT NULL UNIQUE, -- higher specificity will override rules with lower specificity + PRIMARY KEY(`name`) +); + +-- a couple of useful default filter modules +-- these are used in various scripts, so don't remove them if you don't know +-- what you're doing. +-- New filter modules can be added here when needed +INSERT INTO `marc_merge_rules_modules` VALUES('source', 'source from where modification request was sent', 0); +INSERT INTO `marc_merge_rules_modules` VALUES('category', 'categorycode of user who requested modification', 1); +INSERT INTO `marc_merge_rules_modules` VALUES('borrower', 'borrowernumber of user who requested modification', 2); + +CREATE TABLE IF NOT EXISTS `marc_merge_rules` ( + `id` int(11) NOT NULL auto_increment, + `tag` varchar(255) NOT NULL, -- can be regexp, so need > 3 chars + `module` varchar(255) NOT NULL, + `filter` varchar(255) NOT NULL, + `add` tinyint NOT NULL, + `append` tinyint NOT NULL, + `remove` tinyint NOT NULL, + `delete` tinyint NOT NULL, + PRIMARY KEY(`id`), + CONSTRAINT `marc_merge_rules_ibfk1` FOREIGN KEY (`module`) REFERENCES `marc_merge_rules_modules` (`name`) ON DELETE CASCADE +); diff --git a/installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql b/installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql new file mode 100644 index 0000000000..5116c0a355 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql @@ -0,0 +1 @@ +INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES ('MARCMergeRules','0','','Use the MARC merge rules system to decide what actions to take for each field when modifying records.','YesNo'); diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index 974bccf832..90ec53279c 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -40,6 +40,7 @@ [% IF Koha.Preference('SearchEngine') == 'Elasticsearch' %]
  • Search engine configuration
  • [% END %] +
  • MARC merge rules
  • [% IF ( CAN_user_acquisition_period_manage || CAN_user_acquisition_budget_manage || CAN_user_parameters || CAN_user_acquisition_edi_manage ) %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index db9443391e..ec4b3f8196 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -93,6 +93,8 @@
    Search engine configuration
    Manage indexes, facets, and their mappings to MARC fields and subfields.
    [% END %] +
    MARC merge rules
    +
    Managed MARC field merge rules
    [% IF ( CAN_user_acquisition_currencies_manage || CAN_user_acquisition_period_manage || CAN_user_acquisition_budget_manage || CAN_user_acquisition_edi_manage ) %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt new file mode 100644 index 0000000000..8983c29f2e --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt @@ -0,0 +1,395 @@ +[% USE Koha %] +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › MARC merge rules +[% INCLUDE 'doc-head-close.inc' %] + +[% INCLUDE 'datatables.inc' %] + + + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
    +
    +
    +
    + +

    Manage MARC merge rules

    + +[% FOR m IN messages %] +
    + [% SWITCH m.code %] + [% CASE 'invalid_tag_regexp' %] + Invalid regular expression "[% m.tag | html %]". + [% CASE 'invalid_control_field_actions' %] + Invalid combination of actions for tag [% m.tag | html %]. Control field rules do not allow "Appended: Append" and "Removed: Skip". + [% CASE %] + [% m.code | html %] + [% END %] +
    +[% END %] + +[% UNLESS Koha.Preference( 'MARCMergeRules' ) %] +
    + The MARCMergeRules preference is not set, don't forget to enable it for rules to take effect. +
    +[% END %] +[% IF removeConfirm %] +
    +

    Remove rule?

    +

    Are you sure you want to remove the selected rule(s)?

    + +
    + +
    + +
    +[% END %] + +
    + + + + + + + + + + + + + + + [% UNLESS edit %] + + + + + + + + + + + + + + + + [% END %] + + [% FOREACH rule IN rules %] + + [% IF rule.edit %] + + + + + + + + + + + + [% ELSE %] + + + + + + + + + + + + [% END %] + + [% END %] + +
    RuleModuleFilterTagPresetAddedAppendedRemovedDeletedActions 
      + + + + + + + + + + + +
    [% rule.id %] + + + + + + + + + + + + + + + [% rule.id %][% rule.module %][% rule.filter %][% rule.tag %][% IF rule.add %]Add[% ELSE %]Skip[% END %][% IF rule.append %]Append[% ELSE %]Skip[% END %][% IF rule.remove %]Remove[% ELSE %]Skip[% END %][% IF rule.delete %]Delete[% ELSE %]Skip[% END %] + Delete + Edit + + +
    +
    + +
    + +
    + +
    +
    +
    +[% INCLUDE 'admin-menu.inc' %] +
    +
    +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref index d730530764..2ba8f54096 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref @@ -280,3 +280,10 @@ Cataloging: - "All values of repeating tags and subfields will be printed with the given RIS tag." - "
    " - "Use of TY ( record type ) as a key will replace the default TY with the field value of your choosing." + - + - When importing records + - pref: MARCMergeRules + choices: + yes: "use" + no: "don't use" + - MARC merge rules to decide which action to take for each field. diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt index 4d6a214113..79ad0e1883 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/addbiblio.tt @@ -561,6 +561,7 @@ function Changefwk() { [% END %] + diff --git a/misc/migration_tools/bulkmarcimport.pl b/misc/migration_tools/bulkmarcimport.pl index d820d3e991..83fa9a31e4 100755 --- a/misc/migration_tools/bulkmarcimport.pl +++ b/misc/migration_tools/bulkmarcimport.pl @@ -454,7 +454,7 @@ RECORD: while ( ) { $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber; }; if ($update) { - eval { ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber) ) }; + eval { ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber), {context => {source => 'bulkmarcimport'}} ) }; if ($@) { warn "ERROR: Edit biblio $biblionumber failed: $@\n"; printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile); diff --git a/misc/migration_tools/import_lexile.pl b/misc/migration_tools/import_lexile.pl index 131db44b12..73ffc5453d 100755 --- a/misc/migration_tools/import_lexile.pl +++ b/misc/migration_tools/import_lexile.pl @@ -153,6 +153,7 @@ while ( my $row = $csv->getline_hr($fh) ) { foreach my $biblionumber (@biblionumbers) { $counter++; my $record = GetMarcBiblio({ biblionumber => $biblionumber }); + my $frameworkcode = GetFrameworkCode($biblionumber); if ($verbose) { say "Found matching record! Biblionumber: $biblionumber"; @@ -202,7 +203,7 @@ while ( my $row = $csv->getline_hr($fh) ) { $record->append_fields($field); } - ModBiblio( $record, $biblionumber ) unless ( $test ); + ModBiblio( $record, $biblionumber, $frameworkcode, {context => {source => 'import_lexile'}} ) unless ( $test ); } } diff --git a/t/db_dependent/Biblio/MarcMergeRules.t b/t/db_dependent/Biblio/MarcMergeRules.t new file mode 100755 index 0000000000..c24f2258c2 --- /dev/null +++ b/t/db_dependent/Biblio/MarcMergeRules.t @@ -0,0 +1,721 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use Try::Tiny; + +use MARC::Record; + +use C4::Context; +use C4::Biblio; +use Koha::Database; #?? + +use Test::More tests => 22; +use Test::MockModule; + +use Koha::MarcMergeRules; +use Koha::MarcMergeRulesModules; + +use t::lib::Mocks; + +my $schema = Koha::Database->schema; +$schema->storage->txn_begin; + +t::lib::Mocks::mock_preference('MARCMergeRules', '1'); + +# Create a record +my $orig_record = MARC::Record->new(); +$orig_record->append_fields ( + MARC::Field->new('250', '','', 'a' => '250 bottles of beer on the wall'), + MARC::Field->new('250', '','', 'a' => '256 bottles of beer on the wall'), + MARC::Field->new('500', '','', 'a' => 'One bottle of beer in the fridge'), +); + +my $modules_rs = Koha::MarcMergeRulesModules->search(undef, { order_by => { -desc => 'specificity' } }); +my @modules; +push @modules, $modules_rs->next; + +my $incoming_record = MARC::Record->new(); +$incoming_record->append_fields( + MARC::Field->new('250', '', '', 'a' => '256 bottles of beer on the wall'), # Unchanged + MARC::Field->new('250', '', '', 'a' => '251 bottles of beer on the wall'), # Appended + # MARC::Field->new('250', '', '', 'a' => '250 bottles of beer on the wall'), # Removed + # MARC::Field->new('500', '', '', 'a' => 'One bottle of beer in the fridge'), # Deleted + MARC::Field->new('501', '', '', 'a' => 'One cold bottle of beer in the fridge'), # Added + MARC::Field->new('501', '', '', 'a' => 'Two cold bottles of beer in the fridge'), # Added +); + +# Test default behavior when MARCMergeRules is enabled, but no rules defined (overwrite) +subtest 'Record fields has been overwritten when no merge rules are defined' => sub { + plan tests => 4; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +my $rule = Koha::MarcMergeRules->find_or_create({ + tag => '*', + module => $modules[0]->id, + filter => '*', + add => 0, + append => 0, + remove => 0, + delete => 0 +}); + +subtest 'Record fields has been protected when matched merge all rule operations are set to "0"' => sub { + plan tests => 3; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Only new fields has been added when add = 1, append = 0, remove = 0, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields retain their original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field retain it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Only appended fields has been added when add = 0, append = 1, remove = 0, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"251" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + +}; + +subtest 'Appended and added fields has been added when add = 1, append = 1, remove = 0, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 6, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"251" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been only removed when add = 0, append = 0, remove = 1, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Record fields has been added and removed when add = 1, append = 0, remove = 1, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended and removed when add = 0, append = 1, remove = 1, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Record fields has been added, appended and removed when add = 0, append = 1, remove = 1, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been deleted when add = 0, append = 0, remove = 0, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); +}; + +subtest 'Record fields has been added and deleted when add = 1, append = 0, remove = 0, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended and deleted when add = 0, append = 1, remove = 0, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" field has been appended' + ); +}; + +subtest 'Record fields has been added, appended and deleted when add = 1, append = 1, remove = 0, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been removed and deleted when add = 0, append = 0, remove = 1, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 1, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); +}; + +subtest 'Record fields has been added, removed and deleted when add = 1, append = 0, remove = 1, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended, removed and deleted when add = 0, append = 1, remove = 1, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); +}; + +subtest 'Record fields has been overwritten when add = 1, append = 1, remove = 1, delete = 1' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +# Test rule tag specificity + +# Protect field 500 with more specific tag value +my $skip_all_rule = Koha::MarcMergeRules->find_or_create({ + tag => '500', + module => $modules[0]->id, + filter => '*', + add => 0, + append => 0, + remove => 0, + delete => 0 +}); + +subtest '"500" field has been protected when rule matching on tag "500" is add = 0, append = 0, remove = 0, delete = 0' => sub { + plan tests => 4; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +# Test regexp matching +subtest '"5XX" fields has been protected when rule matching on regexp "5\d{2}" is add = 0, append = 0, remove = 0, delete = 0' => sub { + plan tests => 3; + + $skip_all_rule->set( + { + 'tag' => '5\d{2}', + } + ); + $skip_all_rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +# Test module specificity, the 0 all rule should no longer be included in set of applied rules +subtest 'Record fields has been overwritten when non wild card rule with filter match is add = 1, append = 1, remove = 1, delete = 1' => sub { + plan tests => 4; + + $rule->set( + { + 'filter' => 'test', + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'An exception is thrown when append = 1, remove = 0 is set for control field rule' => sub { + plan tests => 2; + my $exception = try { + $rule->set( + { + 'tag' => '008', + 'append' => 1, + 'remove' => 0, + } + ); + } + catch { + return $_; + }; + ok(defined $exception, "Exception was caught"); + ok($exception->isa('Koha::Exceptions::MarcMergeRule::InvalidControlFieldActions'), "Exception is of correct class"); +}; + +subtest 'An exception is thrown when rule tag is set to invalid regexp' => sub { + plan tests => 2; + + my $exception = try { + $rule->set( + { + 'tag' => '**' + } + ); + } + catch { + return $_; + }; + ok(defined $exception, "Exception was caught"); + ok($exception->isa('Koha::Exceptions::MarcMergeRule::InvalidTagRegExp'), "Exception is of correct class"); +}; + +$schema->storage->txn_rollback; + +1; diff --git a/tools/batch_record_modification.pl b/tools/batch_record_modification.pl index 2b7608469b..873b4168d2 100755 --- a/tools/batch_record_modification.pl +++ b/tools/batch_record_modification.pl @@ -159,7 +159,7 @@ if ( $op eq 'form' ) { my ( $job ); if ( $runinbackground ) { my $job_size = scalar( @record_ids ); - $job = C4::BackgroundJob->new( $sessionID, "FIXME", '/cgi-bin/koha/tools/batch_record_modification.pl', $job_size ); + $job = C4::BackgroundJob->new( $sessionID, "FIXME", $ENV{SCRIPT_NAME}, $job_size ); my $job_id = $job->id; if (my $pid = fork) { $dbh->{InactiveDestroy} = 1; @@ -171,7 +171,7 @@ if ( $op eq 'form' ) { } elsif (defined $pid) { close STDOUT; } else { - warn "fork failed while attempting to run tools/batch_record_modification.pl as a background job"; + warn "fork failed while attempting to run $ENV{'SCRIPT_NAME'} as a background job"; exit 0; } } @@ -195,7 +195,14 @@ if ( $op eq 'form' ) { my $record = GetMarcBiblio({ biblionumber => $biblionumber }); ModifyRecordWithTemplate( $mmtid, $record ); my $frameworkcode = C4::Biblio::GetFrameworkCode( $biblionumber ); - ModBiblio( $record, $biblionumber, $frameworkcode ); + my ($member) = Koha::Patrons->find($loggedinuser); + ModBiblio( $record, $biblionumber, $frameworkcode, + { + source => 'batchmod', + category => $member->{'category_type'}, + borrower => $loggedinuser + } + ); }; if ( $error and $error != 1 or $@ ) { # ModBiblio returns 1 if everything as gone well push @messages, { -- 2.11.0