From a31984534adfc1cb2cb6b4e1d90a22a37bd5af47 Mon Sep 17 00:00:00 2001 From: David Gustafsson Date: Tue, 5 Jun 2018 16:59:55 +0200 Subject: [PATCH] Bug 14957 - Refactor and fix tests and documention Change "MARC permissions" to "MARC merge rules" since this is more descriptive of the feature provided. Update help page to reflect changes since large refactorization. Replace outdated tests with up to date tests. Improve rules admin UI adding "preset" setting. Move large part of added C4::Biblio code into Koha:: namespace. --- C4/Biblio.pm | 584 +---------------- Koha/MarcMergeRule.pm | 40 ++ Koha/MarcMergeRules.pm | 291 +++++++++ Koha/MarcMergeRulesModule.pm | 40 ++ Koha/MarcMergeRulesModules.pm | 31 + admin/marc-merge-rules.pl | 144 +++++ admin/marc-permissions.pl | 115 ---- help.pl | 4 +- .../atomicupdate/bug_14957-marc-merge-rules.sql | 30 + .../bug_14957-marc-permissions-syspref.sql | 2 - .../atomicupdate/bug_14957-marc-permissions.sql | 31 - .../atomicupdate/bug_14957-merge-rules-syspref.sql | 1 + .../intranet-tmpl/prog/en/includes/admin-menu.inc | 2 +- .../prog/en/modules/admin/admin-home.tt | 4 +- .../prog/en/modules/admin/marc-merge-rules.tt | 382 ++++++++++++ .../prog/en/modules/admin/marc-permissions.tt | 320 ---------- .../en/modules/admin/preferences/cataloguing.pref | 4 +- .../prog/en/modules/admin/preferences/logs.pref | 6 - .../prog/en/modules/help/marc-merge-rules.tt | 144 +++++ .../prog/en/modules/help/marc-permissions.tt | 148 ----- t/db_dependent/Biblio/MARCPermissions.t | 279 --------- t/db_dependent/Biblio/MarcMergeRules.t | 689 +++++++++++++++++++++ 22 files changed, 1833 insertions(+), 1458 deletions(-) create mode 100644 Koha/MarcMergeRule.pm create mode 100644 Koha/MarcMergeRules.pm create mode 100644 Koha/MarcMergeRulesModule.pm create mode 100644 Koha/MarcMergeRulesModules.pm create mode 100755 admin/marc-merge-rules.pl delete mode 100755 admin/marc-permissions.pl create mode 100644 installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql delete mode 100644 installer/data/mysql/atomicupdate/bug_14957-marc-permissions-syspref.sql delete mode 100644 installer/data/mysql/atomicupdate/bug_14957-marc-permissions.sql create mode 100644 installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt delete mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-permissions.tt create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-merge-rules.tt delete mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-permissions.tt delete mode 100755 t/db_dependent/Biblio/MARCPermissions.t create mode 100755 t/db_dependent/Biblio/MarcMergeRules.t diff --git a/C4/Biblio.pm b/C4/Biblio.pm index efc0797f40..6b871ac142 100644 --- a/C4/Biblio.pm +++ b/C4/Biblio.pm @@ -100,11 +100,11 @@ BEGIN { &CountItemsIssued &CountBiblioInOrders - &GetMarcPermissionsRules - &GetMarcPermissionsModules - &ModMarcPermissionsRule - &AddMarcPermissionsRule - &DelMarcPermissionsRule + &GetMarcMergeRules + &GetMarcMergeRulesModules + &ModMarcMergeRulesRule + &AddMarcMergeRulesRule + &DelMarcMergeRulesRule ); # To modify something @@ -132,7 +132,7 @@ BEGIN { # they are useful in a few circumstances, so they are exported, # but don't use them unless you are a core developer ;-) push @EXPORT, qw( - &ApplyMarcPermissions + &ApplyMarcMergeRules &ModBiblioMarc ); @@ -321,12 +321,12 @@ sub ModBiblio { _strip_item_fields($record, $frameworkcode); - # apply permissions - if (C4::Context->preference('MARCPermissions') && $biblionumber && defined $options && exists $options->{'context'}) { - $record = ApplyMarcPermissions({ + # apply merge rules + if (C4::Context->preference('MARCMergeRules') && $biblionumber && defined $options && exists $options->{'context'}) { + $record = ApplyMarcMergeRules({ biblionumber => $biblionumber, record => $record, - filter => $options->{'context'}, + context => $options->{'context'}, } ); } @@ -3552,13 +3552,13 @@ sub RemoveAllNsb { return $record; } -=head2 ApplyMarcPermissions +=head2 ApplyMarcMergeRules - my $record = ApplyMarcPermissions($arguments) + my $record = ApplyMarcMergeRules($params) -Applies marc permission rules to a record. +Applies marc merge rules to a record. -C<$arguments> is expected to be a hashref with below keys defined. +C<$params> is expected to be a hashref with below keys defined. =over 4 @@ -3566,26 +3566,17 @@ C<$arguments> is expected to be a hashref with below keys defined. biblionumber of old record =item C -record that will modify old record +Incoming record that will be merged with old record -=item C -only tags included in framework will be processed - -=item C -hashref containing at least one filter module from the marc_permissions_modules -table in form {module => filter}. Three predefined filter modules exists: +=item C +hashref containing at least one module from marc_merge_rules_modules table +with filter value on the form {module => filter, ...}. Three predefined +context modules exists: * source * category * borrower -=item C -optional reference to array that will be filled with rule evaluation log -entries. - -=item C -optional boolean which when true disables logging to action log. - =back Returns: @@ -3594,545 +3585,38 @@ Returns: =item C<$record> -new MARC record based on C with C applied. If no old +Merged MARC record based with merge rules for C applied. If no old record for C can be found, C is returned unchanged. -Default action when no matching filter found is to leave old record unchanged. +Default action when no matching context is found to return C unchanged. +If no rules are found for a certain field tag the default is to overwrite with +fields with this field tag from C. =back =cut -sub ApplyMarcPermissions { - my ($arguments) = @_; - my $biblionumber = $arguments->{biblionumber}; - my $incoming_record = $arguments->{record}; +sub ApplyMarcMergeRules { + my ($params) = @_; + my $biblionumber = $params->{biblionumber}; + my $incoming_record = $params->{record}; - if ( !$biblionumber ) { - carp 'ApplyMarcPermissions called on undefined biblionumber'; + if (!$biblionumber) { + carp 'ApplyMarcMergeRules called on undefined biblionumber'; return; } - if ( !$incoming_record ) { - carp 'ApplyMarcPermissions called on undefined record'; + if (!$incoming_record) { + carp 'ApplyMarcMergeRules called on undefined record'; return; } my $old_record = GetMarcBiblio({ biblionumber => $biblionumber }); - my $merge_rules = undef; - if ($old_record && $arguments->{filter} && ($merge_rules = GetMarcPermissions($arguments->{filter}))) { - return MergeRecords($old_record, $incoming_record, $merge_rules); + # Skip merge rules if called with no context + if ($old_record && defined $params->{context}) { + return Koha::MarcMergeRules->merge_records($old_record, $incoming_record, $params->{context}); } return $incoming_record; } -sub MergeRecords { - my ($old_record, $incoming_record, $merge_rules) = @_; - my $is_matching_regex = sub { - my ( $tag, $m ) = @_; - - # tag is not exactly same as possible regex - $tag ne $m && - - # wildcard - $m ne '*' && - - # valid tagDataType - $m !~ /^(0[1-9A-z][\dA-Z]) | - ([1-9A-z][\dA-z]{2})$/x && - - # nor valid controltagDataType - $m !~ /00[1-9A-Za-z]{1}/ && - - # so we try it as a regex - $tag =~ /^$m$/ - }; - - my $fields_by_tag = sub { - my ($record) = @_; - my $fields = {}; - foreach my $field ($record->fields()) { - $fields->{$field->tag()} //= []; - push @{$fields->{$field->tag()}}, $field; - } - return $fields; - }; - - my $hash_field_data = sub { - my ($field) = @_; - my $indicators = join("\x1E", map { $field->indicator($_) } (1, 2)); - return $indicators . "\x1E" . join("\x1E", sort map { join "\x1E", @{$_} } $field->subfields()); - }; - - my $diff_by_key = sub { - my ($a, $b) = @_; - my @removed; - my @intersecting; - my @added; - my %keys_index = map { $_ => undef } (keys %{$a}, keys %{$b}); - foreach my $key (keys %keys_index) { - if ($a->{$key} && $b->{$key}) { - push @intersecting, $a->{$key}; - } - elsif ($a->{$key}) { - push @removed, $a->{$key}; - } - else { - push @added, $b->{$key}; - } - } - return (\@removed, \@intersecting, \@added); - }; - - my $get_matching_field_rule = sub { - my ($tag) = @_; - my $matched_rule = undef; - # Exact match takes precedence - if (exists $merge_rules->{$tag}) { - $matched_rule = $merge_rules->{$tag}; - } - else { - # TODO: sorty by module/weight rule id or something, grab first matching via listutils thingy - my @matching_rules = map { $merge_rules->{$_} } grep { $is_matching_regex->($tag, $_) } sort keys %{$merge_rules}; - # TODO: fix - if (@matching_rules) { - $matched_rule = pop @matching_rules; - } - elsif($merge_rules->{'*'}) { - $matched_rule = $merge_rules->{'*'}; - } - } - return $matched_rule; - }; - - my $merged_record = MARC::Record->new(); - my @merged_record_fields; - - # Leader is always overwritten, or kept??? - $merged_record->leader($incoming_record->leader()); - - my $current_fields = $fields_by_tag->($old_record); - my $incoming_fields = $fields_by_tag->($incoming_record); - - # First we get all new incoming control fields - my @new_field_tags = grep { !(exists $current_fields->{$_}) } keys %{$incoming_fields}; - - foreach my $tag (@new_field_tags) { - my $rule = $get_matching_field_rule->($tag) // { - on_new => {'action' => 'skip', 'rule' => 0} - }; - if ( - $rule->{on_new}->{action} eq 'add' || - $rule->{on_new}->{action} eq 'overwrite' # ??? - ) { # Or could just be write/protect? - # Hmm, only one control field possible?? - push @merged_record_fields, @{$incoming_fields->{$tag}}; - } - } - - # Then we get all control fields no longer present in incoming fields - # (removed) - my @deleted_field_tags = grep { !(exists $incoming_fields->{$_}) } keys %{$current_fields}; - foreach my $tag (@deleted_field_tags) { - my $rule = $get_matching_field_rule->($tag) // { - on_deleted => {'action' => 'skip', 'rule' => 0} - }; - if ($rule->{on_deleted}->{action} eq 'skip') { - push @merged_record_fields, @{$current_fields->{$tag}}; - } - } - - # Then we get the intersection of control fields, present both in - # current and incoming record (possibly to be overwritten) - my @common_field_tags = grep { exists $incoming_fields->{$_} } keys %{$current_fields}; - foreach my $tag (@common_field_tags) { - # Is control field - my $rule = $get_matching_field_rule->($tag) // { - on_removed => {'action' => 'skip', 'rule' => 0}, - on_appended => {'action' => 'skip', 'rule' => 0} - }; - if ($tag < 10) { - # on_existing = on_match - if ($rule->{on_appended}->{action} eq 'skip') { # TODO: replace with "protect", "keep" - push @merged_record_fields, @{$current_fields->{$tag}}; - } - elsif ($rule->{on_appended}->{action} eq 'append') { - push @merged_record_fields, @{$incoming_fields->{$tag}}; - } - if ( - $rule->{on_appended}->{action} eq 'append' && - $rule->{on_removed}->{action} eq 'skip' - ) { - #TODO: This is an invalid combination for control fields, warn!! - # Or should perform client/server side validation to prevent this choice - } - } - else { - # Compute intersection and diff using field data - my %current_fields_by_data = map { $hash_field_data->($_) => $_ } @{$current_fields->{$tag}}; - my %incoming_fields_by_data = map { $hash_field_data->($_) => $_ } @{$incoming_fields->{$tag}}; - my ($current_fields_only, $common_fields, $incoming_fields_only) = $diff_by_key->(\%current_fields_by_data, \%incoming_fields_by_data); - - # First add common fields (intersection) - # Unchanged - if (@{$common_fields}) { - push @merged_record_fields, @{$common_fields}; - } - # Removed - if (@{$current_fields_only}) { - if ($rule->{on_removed}->{action} eq 'skip') { - push @merged_record_fields, @{$current_fields_only}; - } - } - # Appended - if (@{$incoming_fields_only}) { - if ($rule->{on_appended}->{action} eq 'append') { - push @merged_record_fields, @{$incoming_fields_only}; - } - } - } - } - if ($#merged_record_fields != 0) { - $merged_record->insert_fields_ordered(@merged_record_fields); - } - return $merged_record; -} - -=head2 GetMarcPermissions - - my $marc_permissions = GetMarcPermissions() - -Loads MARC field permissions from the marc_permissions table. - -Returns: - -=over 4 - -=item C<$marc_permissions> - -hashref with permissions structure for use with GetMarcPermissionsAction. - -=back - -=cut - -sub GetMarcPermissions { - my ($filter) = @_; - my $dbh = C4::Context->dbh; - my $rule_count = 0; - my $modules = GetMarcPermissionsModules(); - # We only care about modules included in the context/filter - # TODO: Perhaps make sure source => '*' is default? - my @filter_modules = grep { exists $filter->{$_->{name}} } @{$modules}; - - my $cache = Koha::Caches->get_instance(); - my $permissions = $cache->get_from_cache('marc_permissions', { unsafe => 1 }); - - if (!$permissions) { - my $query = ' - SELECT `marc_permissions`.*, - `marc_permissions_modules`.`name`, - `marc_permissions_modules`.`description`, - `marc_permissions_modules`.`specificity` - FROM `marc_permissions` - LEFT JOIN `marc_permissions_modules` ON `module` = `marc_permissions_modules`.`id` - ORDER BY `marc_permissions_modules`.`specificity`, `id` - '; - my $sth = $dbh->prepare($query); - $sth->execute(); - while (my $perm = $sth->fetchrow_hashref) { - my $target = ($permissions->{$perm->{name}}->{$perm->{filter}}->{$perm->{tagfield}} //= {}); - foreach my $event (GetMarcPermissionEvents()) { - $target->{$event} = { action => $perm->{$event}, rule => $perm->{'id'} }; - } - } - $cache->set_in_cache('marc_permissions', $permissions); - } - - my $filtered_permissions = undef; - foreach my $module (@filter_modules) { - if ( - exists $permissions->{$module->{name}} && - exists $permissions->{$module->{name}}->{$filter->{$module->{name}}} - ) { - # TODO: Support multiple overlapping filters/context?? - $filtered_permissions = $permissions->{$module->{name}}->{$filter->{$module->{name}}}; - last; - } - } - if (!$filtered_permissions) { - # No perms matching specific context conditions found, try wildcard value for each active context - foreach my $module (@filter_modules) { - if (exists $permissions->{$module->{name}}->{'*'}) { - $filtered_permissions = $permissions->{$module->{name}}->{'*'}; - last; - } - } - } - return $filtered_permissions; -} - -# TODO: Use this in GetMarcPermissions -=head2 GetMarcPermissionsRules - - my $rules = GetMarcPermissionsRules() - -Returns: - -=over 4 - -=item C<$rules> - -array (in list context, arrayref otherwise) of hashrefs from marc_permissions -table in order of module specificity and rule id. - -=back - -=cut - -sub GetMarcPermissionsRules { - my $dbh = C4::Context->dbh; - my @rules = (); - - my $query = ' - SELECT `marc_permissions`.`id`, - `marc_permissions`.`tagfield`, - `marc_permissions`.`filter`, - `marc_permissions`.`on_new`, - `marc_permissions`.`on_appended`, - `marc_permissions`.`on_removed`, - `marc_permissions`.`on_deleted`, - `marc_permissions_modules`.`name` as `module`, - `marc_permissions_modules`.`description`, - `marc_permissions_modules`.`specificity` - FROM `marc_permissions` - LEFT JOIN `marc_permissions_modules` ON `module` = `marc_permissions_modules`.`id` - ORDER BY `marc_permissions_modules`.`specificity`, `id` - '; - my $sth = $dbh->prepare($query); - $sth->execute(); - while ( my $row = $sth->fetchrow_hashref ) { - push(@rules, $row); - } - - return wantarray ? @rules : \@rules; -} - -=head2 GetMarcPermissionsModules - - my $modules = GetMarcPermissionsModules() - -Returns: - -=over 4 - -=item C<$modules> - -array (in list context, arrayref otherwise) of hashrefs from -marc_permissions_modules table in order of specificity. - -=back - -=cut - -sub GetMarcPermissionsModules { - my $dbh = C4::Context->dbh; - my @modules = (); - - my $query = ' - SELECT * - FROM `marc_permissions_modules` - ORDER BY `specificity` DESC - '; - my $sth = $dbh->prepare($query); - $sth->execute(); - while ( my $row = $sth->fetchrow_hashref ) { - push(@modules, $row); - } - - return wantarray ? @modules : \@modules; -} - -sub GetMarcPermissionEvents { - return ('on_new', 'on_appended', 'on_removed', 'on_deleted'); -} - -=head2 ModMarcPermissionsRule - - my $success = ModMarcPermissionsRule($id, $fields) - -Modifies rule in the marc_permissions table. - -=over 4 - -=item C<$id> - -rule id to modify - -=item C<$fields> - -hashref defining the table fields - - * tagfield - required - * module - required - * filter - required - * on_new - required - * on_appended - required - * on_removed - required - * on_deleted - required - -=back - -Returns: - -=over 4 - -=item C<$success> - -undef if an error occurs, otherwise true. - -=back - -=cut - -sub ModMarcPermissionsRule { - my ($id, $f) = @_; - my $dbh = C4::Context->dbh; - - my $query = ' - UPDATE `marc_permissions` - SET - tagfield = ?, - module = ?, - filter = ?, - on_new = ?, - on_appended = ?, - on_removed = ?, - on_deleted = ? - WHERE - id = ? - '; - my $sth = $dbh->prepare($query); - my $result = $sth->execute ( - $f->{tagfield}, - $f->{module}, - $f->{filter}, - $f->{on_new}, - $f->{on_appended}, - $f->{on_removed}, - $f->{on_deleted}, - $id - ); - ClearMarcPermissionsRulesCache(); - return $result; -} - -sub ClearMarcPermissionsRulesCache { - my $cache = Koha::Caches->get_instance(); - $cache->clear_from_cache('marc_permissions'); -} - -=head2 AddMarcPermissionsRule - - my $success = AddMarcPermissionsRule($fields) - -Add rule to the marc_permissions table. - -=over 4 - -=item C<$fields> - -hashref defining the table fields - - tagfield - required - module - required - filter - required - on_new - required - on_appended - required - on_removed - required - on_deleted - required - -=back - -Returns: - -=over 4 - -=item C<$success> - -undef if an error occurs, otherwise true. - -=back - -=cut - -sub AddMarcPermissionsRule { - my $f = shift; - my $dbh = C4::Context->dbh; - my $query = ' - INSERT INTO `marc_permissions` - ( - tagfield, - module, - filter, - on_new, - on_appended, - on_removed, - on_deleted - ) - VALUES (?, ?, ?, ?, ?, ?, ?) - '; - my $sth = $dbh->prepare($query); - my $result = $sth->execute ( - $f->{tagfield}, - $f->{module}, - $f->{filter}, - $f->{on_new}, - $f->{on_appended}, - $f->{on_removed}, - $f->{on_deleted} - ); - ClearMarcPermissionsRulesCache(); - return $result; -} - -=head2 DelMarcPermissionsRule - - my $success = DelMarcPermissionsRule($id) - -Deletes rule from the marc_permissions table. - -=over 4 - -=item C<$id> - -rule id to delete - -=back - -Returns: - -=over 4 - -=item C<$success> - -undef if an error occurs, otherwise true. - -=back - -=cut - -sub DelMarcPermissionsRule { - my $id = shift; - my $dbh = C4::Context->dbh; - my $query = ' - DELETE FROM `marc_permissions` - WHERE - id = ? - '; - my $sth = $dbh->prepare($query); - my $result = $sth->execute($id); - ClearMarcPermissionsRulesCache(); - return $result; -} 1; __END__ diff --git a/Koha/MarcMergeRule.pm b/Koha/MarcMergeRule.pm new file mode 100644 index 0000000000..7e42e292b5 --- /dev/null +++ b/Koha/MarcMergeRule.pm @@ -0,0 +1,40 @@ +package Koha::MarcMergeRule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use parent qw(Koha::Object); + +my $cache = Koha::Caches->get_instance(); + +=head1 NAME + +Koha::MarcMergeRule - Koha SearchField Object class + +=cut + +sub store { + my $self = shift @_; + $cache->clear_from_cache('marc_merge_rules'); + $self->SUPER::store(@_); +} + +sub _type { + return 'MarcMergeRule'; +} + +1; diff --git a/Koha/MarcMergeRules.pm b/Koha/MarcMergeRules.pm new file mode 100644 index 0000000000..a58c0b9c07 --- /dev/null +++ b/Koha/MarcMergeRules.pm @@ -0,0 +1,291 @@ +package Koha::MarcMergeRules; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use List::Util qw(first); +use Koha::MarcMergeRule; + +use parent qw(Koha::Objects); + +sub operations { + return ('add', 'append', 'remove', 'delete'); +} + +my $cache = Koha::Caches->get_instance(); + +=head1 NAME + +Koha::MarcMergeRules - Koha MarcMergeRules Object set class + +=head1 API + +=head2 Class Methods + +=head3 context_rules + + my $rules = Koha::MarcMergeRules->context_rules($context); + +Gets all MARC merge rules for the supplied C<$context> (hashref with { module => filter, ... } values). + +=cut + +sub context_rules { + my ($self, $context) = @_; + + return unless %{$context}; + + my $rules = $cache->get_from_cache('marc_merge_rules', { unsafe => 1 }); + + if (!$rules || 1) { + $rules = {}; + my @rules_rows = $self->_resultset()->search( + undef, + { + prefetch => 'module', + order_by => { -desc => [qw/module.specificity me.id/] } + } + ); + foreach my $rule_row (@rules_rows) { + my %rule = $rule_row->get_columns(); + my $operations = {}; + + foreach my $operation ($self->operations) { + $operations->{$operation} = { allow => $rule{$operation}, rule => $rule{id} }; + } + + # TODO: Remove unless check and validate on saving rules? + if ($rule{tag} eq '*') { + unless (exists $rules->{$rule{module}}->{$rule{filter}}->{'*'}) { + $rules->{$rule{module}}->{$rule{filter}}->{'*'} = $operations; + } + } + elsif ($rule{tag} =~ /^(\d{3})$/) { + unless (exists $rules->{$rule{module}}->{$rule{filter}}->{tags}->{$rule{tag}}) { + $rules->{$rule{module}}->{$rule{filter}}->{tags}->{$rule{tag}} = $operations; + } + } + else { + my $regexps = ($rules->{$rule{module}}->{$rule{filter}}->{regexps} //= []); + push @{$regexps}, [$rule{tag}, $operations]; + } + } + $cache->set_in_cache('marc_merge_rules', $rules); + } + + my $context_rules = undef; + foreach my $module_name (keys %{$context}) { + if ( + exists $rules->{$module_name} && + exists $rules->{$module_name}->{$context->{$module_name}} + ) { + $context_rules = $rules->{$module_name}->{$context->{$module_name}}; + last; + } + } + if (!$context_rules) { + # No perms matching specific context conditions found, try wildcard value for each active context + foreach my $module_name (keys %{$context}) { + if (exists $rules->{$module_name}->{'*'}) { + $context_rules = $rules->{$module_name}->{'*'}; + last; + } + } + } + return $context_rules; +} + +=head3 merge_records + + my $merged_record = Koha::MarcMergeRules->merge_records($old_record, $incoming_record, $context); + +Merge C<$old_record> with C<$incoming_record> applying merge rules for C<$context>. +Returns merged record C<$merged_record>. C<$old_record>, C<$incoming_record> and +C<$merged_record> are all MARC::Record objects. + +=cut + +sub merge_records { + my ($self, $old_record, $incoming_record, $context) = @_; + + my $rules = $self->context_rules($context); + + # Default when no rules found is to overwrite with incoming record + return $incoming_record unless $rules; + + my $fields_by_tag = sub { + my ($record) = @_; + my $fields = {}; + foreach my $field ($record->fields()) { + $fields->{$field->tag()} //= []; + push @{$fields->{$field->tag()}}, $field; + } + return $fields; + }; + + my $hash_field_data = sub { + my ($field) = @_; + my $indicators = join("\x1E", map { $field->indicator($_) } (1, 2)); + return $indicators . "\x1E" . join("\x1E", sort map { join "\x1E", @{$_} } $field->subfields()); + }; + + my $diff_by_key = sub { + my ($a, $b) = @_; + my @removed; + my @intersecting; + my @added; + my %keys_index = map { $_ => undef } (keys %{$a}, keys %{$b}); + foreach my $key (keys %keys_index) { + if ($a->{$key} && $b->{$key}) { + push @intersecting, $a->{$key}; + } + elsif ($a->{$key}) { + push @removed, $a->{$key}; + } + else { + push @added, $b->{$key}; + } + } + return (\@removed, \@intersecting, \@added); + }; + + my $tag_rules = $rules->{tags} // {}; + my $default_rule = $rules->{'*'} // { + add => { allow => 1, 'rule' => 0}, + append => { allow => 1, 'rule' => 0}, + delete => { allow => 1, 'rule' => 0}, + remove => { allow => 1, 'rule' => 0}, + }; + + # Precompile regexps + my @regexp_rules = map { { regexp => qr/^$_->[0]$/, actions => $_->[1] } } @{$rules->{regexps} // []}; + + my $get_matching_field_rule = sub { + my ($tag) = @_; + # Exact match takes precedence, then regexp, then wildcard/defaults + return $tag_rules->{$tag} // + %{(first { $tag =~ $_->{regexp} } @regexp_rules) // {}}{actions} // + $default_rule; + }; + + my %merged_record_fields; + + my $current_fields = $fields_by_tag->($old_record); + my $incoming_fields = $fields_by_tag->($incoming_record); + + # First we get all new incoming fields + my @new_field_tags = grep { !(exists $current_fields->{$_}) } keys %{$incoming_fields}; + foreach my $tag (@new_field_tags) { + my $rule = $get_matching_field_rule->($tag); + if ($rule->{add}->{allow}) { + $merged_record_fields{$tag} //= []; + push @{$merged_record_fields{$tag}}, @{$incoming_fields->{$tag}}; + } + } + + # Then we get all fields no longer present in incoming fields + my @deleted_field_tags = grep { !(exists $incoming_fields->{$_}) } keys %{$current_fields}; + foreach my $tag (@deleted_field_tags) { + my $rule = $get_matching_field_rule->($tag); + if (!$rule->{delete}->{allow}) { + $merged_record_fields{$tag} //= []; + push @{$merged_record_fields{$tag}}, @{$current_fields->{$tag}}; + } + } + + # Then we get the intersection of control fields, present both in + # current and incoming record (possibly to be overwritten) + my @common_field_tags = grep { exists $incoming_fields->{$_} } keys %{$current_fields}; + foreach my $tag (@common_field_tags) { + my $rule = $get_matching_field_rule->($tag); + # Compute intersection and diff using field data + my $sort_weight = 0; + my %current_fields_by_data = map { $hash_field_data->($_) => [$sort_weight++, $_] } @{$current_fields->{$tag}}; + + # Always put incoming fields after current fields + my %incoming_fields_by_data = map { $hash_field_data->($_) => [$sort_weight++, $_] } @{$incoming_fields->{$tag}}; + + my ($current_fields_only, $common_fields, $incoming_fields_only) = $diff_by_key->(\%current_fields_by_data, \%incoming_fields_by_data); + + my @merged_fields; + + # First add common fields (intersection) + # Unchanged + if (@{$common_fields}) { + push @merged_fields, @{$common_fields}; + } + # Removed + if (@{$current_fields_only}) { + if (!$rule->{remove}->{allow}) { + push @merged_fields, @{$current_fields_only}; + } + } + # Appended + if (@{$incoming_fields_only}) { + if ($rule->{append}->{allow}) { + push @merged_fields, @{$incoming_fields_only}; + } + } + $merged_record_fields{$tag} //= []; + + # Sort ascending according to weight (original order) + push @{$merged_record_fields{$tag}}, map { $_->[1] } sort { $a->[0] <=> $b->[0] } @merged_fields; + } + + my $merged_record = MARC::Record->new(); + + # Leader is always overwritten, or kept??? + $merged_record->leader($incoming_record->leader()); + + if (%merged_record_fields) { + foreach my $tag (sort keys %merged_record_fields) { + $merged_record->append_fields(@{$merged_record_fields{$tag}}); + } + } + return $merged_record; +} + +sub _clear_caches { + $cache->clear_from_cache('marc_merge_rules'); +} + +sub find_or_create { + my $self = shift @_; + $self->_clear_caches(); + $self->SUPER::find_or_create(@_); +} + +sub update { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::update(@_); +} + +sub delete { + my $self = shift @_; + $self->_clear_caches(); + return $self->SUPER::delete(@_); +} + +sub _type { + return 'MarcMergeRule'; +} + +sub object_class { + return 'Koha::MarcMergeRule'; +} + +1; diff --git a/Koha/MarcMergeRulesModule.pm b/Koha/MarcMergeRulesModule.pm new file mode 100644 index 0000000000..9695759dc9 --- /dev/null +++ b/Koha/MarcMergeRulesModule.pm @@ -0,0 +1,40 @@ +package Koha::MarcMergeRulesModule; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; + +use parent qw(Koha::Object); + +=head1 NAME + +Koha::MarcMergeRulesModule - Koha SearchField Object class + +=head1 API + +=head2 Class Methods + +=cut + +=head3 type + +=cut + +sub _type { + return 'MarcMergeRulesModule'; +} + +1; diff --git a/Koha/MarcMergeRulesModules.pm b/Koha/MarcMergeRulesModules.pm new file mode 100644 index 0000000000..6c1b163699 --- /dev/null +++ b/Koha/MarcMergeRulesModules.pm @@ -0,0 +1,31 @@ +package Koha::MarcMergeRulesModules; + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Koha::MarcMergeRulesModule; + +use parent qw(Koha::Objects); + +sub _type { + return 'MarcMergeRulesModule'; +} + +sub object_class { + return 'Koha::MarcMergeRulesModule'; +} + +1; diff --git a/admin/marc-merge-rules.pl b/admin/marc-merge-rules.pl new file mode 100755 index 0000000000..5f62fb3d34 --- /dev/null +++ b/admin/marc-merge-rules.pl @@ -0,0 +1,144 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use strict; +use warnings; + +# standard or CPAN modules used +use CGI qw ( -utf8 ); +use CGI::Cookie; +use MARC::File::USMARC; + +# Koha modules used +use C4::Context; +use C4::Koha; +use C4::Auth; +use C4::AuthoritiesMarc; +use C4::Output; +use C4::Biblio; +use C4::ImportBatch; +use C4::Matcher; +use C4::BackgroundJob; +use C4::Labels::Batch; +use Koha::MarcMergeRules; +use Koha::MarcMergeRule; +use Koha::MarcMergeRulesModules; + +my $script_name = "/cgi-bin/koha/admin/marc-merge-rules.pl"; + +my $input = new CGI; +my $op = $input->param('op') || ''; +my $errors = []; + +my $rule_from_cgi = sub { + my ($cgi) = @_; + + my %rule = map { $_ => scalar $cgi->param($_) } ( + 'tag', + 'module', + 'filter', + 'add', + 'append', + 'remove', + 'delete' + ); + + my $id = $cgi->param('id'); + if ($id) { + $rule{id} = $id; + } + + return \%rule; +}; + +my ($template, $loggedinuser, $cookie) = get_template_and_user( + { + template_name => "admin/marc-merge-rules.tt", + query => $input, + type => "intranet", + authnotrequired => 0, + flagsrequired => { parameters => 'parameters_remaining_permissions' }, + debug => 1, + } +); + +my %cookies = parse CGI::Cookie($cookie); +our $sessionID = $cookies{'CGISESSID'}->value; + +my $get_rules = sub { + # TODO: order? + return [map { { $_->get_columns() } } Koha::MarcMergeRules->_resultset->all]; +}; +my $rules; + +if ($op eq 'remove' || $op eq 'doremove') { + my @remove_ids = $input->multi_param('batchremove'); + push @remove_ids, scalar $input->param('id') if $input->param('id'); + if ($op eq 'remove') { + $template->{VARS}->{removeConfirm} = 1; + my %remove_ids = map { $_ => undef } @remove_ids; + $rules = $get_rules->(); + for my $rule (@{$rules}) { + $rule->{'removemarked'} = 1 if exists $remove_ids{$rule->{id}}; + } + } + elsif ($op eq 'doremove') { + my @remove_ids = $input->multi_param('batchremove'); + push @remove_ids, scalar $input->param('id') if $input->param('id'); + Koha::MarcMergeRules->search({ id => { in => \@remove_ids } })->delete(); + $rules = $get_rules->(); + } +} +elsif ($op eq 'edit') { + $template->{VARS}->{edit} = 1; + my $id = $input->param('id'); + $rules = $get_rules->(); + for my $rule(@{$rules}) { + if ($rule->{id} == $id) { + $rule->{'edit'} = 1; + last; + } + } +} +elsif ($op eq 'doedit' || $op eq 'add') { + my $rule_data = $rule_from_cgi->($input); + if ($rule_data->{tag} ne '*') { + eval { qr/$rule_data->{tag}/ }; + if ($@) { + push @{$errors}, { + type => 'error', + code => 'invalid_tag_regexp', + tag => $rule_data->{tag}, + message => $@ + }; + } + } + if (!@{$errors}) { + my $rule = Koha::MarcMergeRules->find_or_create($rule_data); + $rule->set($rule_data); + $rule->store(); + $rules = $get_rules->(); + } +} +else { + $rules = $get_rules->(); +} + +my $modules = Koha::MarcMergeRulesModules->as_list; +$template->param( rules => $rules, modules => $modules, messages => $errors ); + +output_html_with_http_headers $input, $cookie, $template->output; diff --git a/admin/marc-permissions.pl b/admin/marc-permissions.pl deleted file mode 100755 index 688b76b14c..0000000000 --- a/admin/marc-permissions.pl +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/perl - -# This file is part of Koha. -# -# Koha is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# Koha is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Koha; if not, see . - -use strict; -use warnings; - -# standard or CPAN modules used -use CGI qw ( -utf8 ); -use CGI::Cookie; -use MARC::File::USMARC; - -# Koha modules used -use C4::Context; -use C4::Koha; -use C4::Auth; -use C4::AuthoritiesMarc; -use C4::Output; -use C4::Biblio; -use C4::ImportBatch; -use C4::Matcher; -use C4::BackgroundJob; -use C4::Labels::Batch; - -my $script_name = "/cgi-bin/koha/admin/marc-permissions.pl"; - -my $input = new CGI; -my $op = $input->param('op') || ''; - -my $rule_from_cgi = sub { - my ($cgi) = @_; - my %rule = map { $_ => scalar $cgi->param($_) } ( - 'tagfield', - 'module', - 'filter', - 'on_new', - 'on_appended', - 'on_removed', - 'on_deleted' - ); - return \%rule; -}; - -my ( $template, $loggedinuser, $cookie ) = get_template_and_user( - { - template_name => "admin/marc-permissions.tt", - query => $input, - type => "intranet", - authnotrequired => 0, - flagsrequired => { parameters => 'parameters_remaining_permissions' }, - debug => 1, - } -); - -my %cookies = parse CGI::Cookie($cookie); -our $sessionID = $cookies{'CGISESSID'}->value; - -my $rules; -if ( $op eq "remove" ) { - $template->{VARS}->{removeConfirm} = 1; - my @removeIDs = $input->multi_param('batchremove'); - push( @removeIDs, scalar $input->param('id') ) if $input->param('id'); - - $rules = GetMarcPermissionsRules(); - for my $removeID (@removeIDs) { - map { $_->{'remove'} = 1 if $_->{'id'} == $removeID } @{$rules}; - } -} -elsif ( $op eq "doremove" ) { - my @removeIDs = $input->multi_param('batchremove'); - push( @removeIDs, scalar $input->param('id') ) if $input->param('id'); - for my $removeID (@removeIDs) { - DelMarcPermissionsRule($removeID); - } - $rules = GetMarcPermissionsRules(); -} -elsif ( $op eq "edit" ) { - $template->{VARS}->{edit} = 1; - my $id = $input->param('id'); - $rules = GetMarcPermissionsRules(); - map { $_->{'edit'} = 1 if $_->{'id'} == $id } @{$rules}; - -} -elsif ( $op eq "doedit" ) { - my $id = $input->param('id'); - my $rule = $rule_from_cgi->($input); - ModMarcPermissionsRule($id, $rule); - $rules = GetMarcPermissionsRules(); -} -elsif ( $op eq "add" ) { - my $rule = $rule_from_cgi->($input); - AddMarcPermissionsRule($rule); - $rules = GetMarcPermissionsRules(); - -} -else { - $rules = GetMarcPermissionsRules(); -} -my $modules = GetMarcPermissionsModules(); -$template->param( rules => $rules, modules => $modules ); - -output_html_with_http_headers $input, $cookie, $template->output; diff --git a/help.pl b/help.pl index 7ca709224f..6b4b859bca 100755 --- a/help.pl +++ b/help.pl @@ -75,8 +75,8 @@ if ( $help_version =~ m|^(\d+)\.(\d{2}).*$| ) { } $template->param( helpVersion => $help_version ); -my $rules = GetMarcPermissionsRules(); -my $modules = GetMarcPermissionsModules(); +my $rules = GetMarcMergeRules(); +my $modules = GetMarcMergeRulesModules(); $template->param( rules => $rules, modules => $modules ); output_html_with_http_headers $query, "", $template->output; diff --git a/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql b/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql new file mode 100644 index 0000000000..b2d99ee3e7 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_14957-marc-merge-rules.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS `marc_merge_rules`; +DROP TABLE IF EXISTS `marc_merge_rules_modules`; + +CREATE TABLE `marc_merge_rules_modules` ( + `name` varchar(255) NOT NULL, + `description` varchar(255), + `specificity` int(11) NOT NULL UNIQUE, -- higher specificity will override rules with lower specificity + PRIMARY KEY(`name`) +); + +-- a couple of useful default filter modules +-- these are used in various scripts, so don't remove them if you don't know +-- what you're doing. +-- New filter modules can be added here when needed +INSERT INTO `marc_merge_rules_modules` VALUES('source', 'source from where modification request was sent', 0); +INSERT INTO `marc_merge_rules_modules` VALUES('category', 'categorycode of user who requested modification', 1); +INSERT INTO `marc_merge_rules_modules` VALUES('borrower', 'borrowernumber of user who requested modification', 2); + +CREATE TABLE IF NOT EXISTS `marc_merge_rules` ( + `id` int(11) NOT NULL auto_increment, + `tag` varchar(255) NOT NULL, -- can be regexp, so need > 3 chars + `module` varchar(255) NOT NULL, + `filter` varchar(255) NOT NULL, + `add` tinyint NOT NULL, + `append` tinyint NOT NULL, + `remove` tinyint NOT NULL, + `delete` tinyint NOT NULL, + PRIMARY KEY(`id`), + CONSTRAINT `marc_merge_rules_ibfk1` FOREIGN KEY (`module`) REFERENCES `marc_merge_rules_modules` (`name`) ON DELETE CASCADE +); diff --git a/installer/data/mysql/atomicupdate/bug_14957-marc-permissions-syspref.sql b/installer/data/mysql/atomicupdate/bug_14957-marc-permissions-syspref.sql deleted file mode 100644 index b63f46ed7b..0000000000 --- a/installer/data/mysql/atomicupdate/bug_14957-marc-permissions-syspref.sql +++ /dev/null @@ -1,2 +0,0 @@ -INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES ('MARCPermissions','0','','Use the MARC permissions system to decide what actions to take for each field when modifying records.','YesNo'); -INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES ('MARCPermissionsLog','0','','Write MARC permissions rule evaluations to the system log when applying rules for MARC field modifications.','YesNo'); diff --git a/installer/data/mysql/atomicupdate/bug_14957-marc-permissions.sql b/installer/data/mysql/atomicupdate/bug_14957-marc-permissions.sql deleted file mode 100644 index c2c2f0720f..0000000000 --- a/installer/data/mysql/atomicupdate/bug_14957-marc-permissions.sql +++ /dev/null @@ -1,31 +0,0 @@ --- DROP TABLE IF EXISTS `marc_permissions`; -DROP TABLE IF EXISTS `marc_permissions_modules`; - -CREATE TABLE `marc_permissions_modules` ( - `id` int(11) NOT NULL auto_increment, - `name` varchar(24) NOT NULL, - `description` varchar(255), - `specificity` int(11) NOT NULL DEFAULT 0, -- higher specificity will override rules with lower specificity - PRIMARY KEY(`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; - --- a couple of useful default filter modules --- these are used in various scripts, so don't remove them if you don't know --- what you're doing. --- New filter modules can be added here when needed -INSERT INTO `marc_permissions_modules` VALUES(NULL, 'source', 'source from where modification request was sent', 0); -INSERT INTO `marc_permissions_modules` VALUES(NULL, 'category', 'categorycode of user who requested modification', 1); -INSERT INTO `marc_permissions_modules` VALUES(NULL, 'borrower', 'borrowernumber of user who requested modification', 2); - -CREATE TABLE IF NOT EXISTS `marc_permissions` ( - `id` int(11) NOT NULL auto_increment, - `tagfield` varchar(255) NOT NULL, -- can be regexe, so need > 3 chars - `module` int(11) NOT NULL, - `filter` varchar(255) NOT NULL, - `on_new` ENUM('skip', 'add') DEFAULT NULL, - `on_appended` ENUM('skip', 'append') DEFAULT NULL, - `on_removed` ENUM('skip', 'remove') DEFAULT NULL, - `on_deleted` ENUM('skip', 'delete') DEFAULT NULL, - PRIMARY KEY(`id`), - CONSTRAINT `marc_permissions_ibfk1` FOREIGN KEY (`module`) REFERENCES `marc_permissions_modules` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; diff --git a/installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql b/installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql new file mode 100644 index 0000000000..5116c0a355 --- /dev/null +++ b/installer/data/mysql/atomicupdate/bug_14957-merge-rules-syspref.sql @@ -0,0 +1 @@ +INSERT IGNORE INTO systempreferences ( `variable`, `value`, `options`, `explanation`, `type` ) VALUES ('MARCMergeRules','0','','Use the MARC merge rules system to decide what actions to take for each field when modifying records.','YesNo'); diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc index 3b2a4fea38..186d4dbd7a 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/admin-menu.inc @@ -40,7 +40,7 @@ [% IF Koha.Preference('SearchEngine') == 'Elasticsearch' %]
  • Search engine configuration
  • [% END %] -
  • MARC field permissions
  • +
  • MARC merge rules
  • Acquisition parameters
    diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt index 5fb10163e4..b0efeb9df1 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/admin-home.tt @@ -93,8 +93,8 @@
    Search engine configuration
    Manage indexes, facets, and their mappings to MARC fields and subfields.
    [% END %] -
    MARC field permissions
    -
    Managed MARC field permissions
    +
    MARC merge rules
    +
    Managed MARC field merge rules

    Acquisition parameters

    diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt new file mode 100644 index 0000000000..3fcedf2086 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-merge-rules.tt @@ -0,0 +1,382 @@ +[% USE Koha %] +[% INCLUDE 'doc-head-open.inc' %] +Koha › Administration › MARC merge rules +[% INCLUDE 'doc-head-close.inc' %] + +[% INCLUDE 'datatables.inc' %] + + + + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
    +
    +
    +
    + +

    Manage MARC merge rules

    + +[% UNLESS Koha.Preference( 'MARCMergeRules' ) %] +
    + The MARCMergeRules preference is not set, don't forget to enable it for rules to take effect. +
    +[% END %] +[% IF removeConfirm %] +
    +

    Remove rule?

    +

    Are you sure you want to remove the selected rule(s)?

    + +
    + +
    + +
    +[% END %] + +
    + + + + + + + + + + + + + + + [% UNLESS edit %] + + + + + + + + + + + + + + + + [% END %] + + [% FOREACH rule IN rules %] + + [% IF rule.edit %] + + + + + + + + + + + + [% ELSE %] + + + + + + + + + + + + [% END %] + + [% END %] + +
    RuleModuleFilterTagPresetAddedAppendedRemovedDeletedActions 
      + + + + + + + + + + + +
    [% rule.id %] + + + + + + + + + + + + + + + [% rule.id %][% rule.module %][% rule.filter %][% rule.tag %][% IF rule.add %]Add[% ELSE %]Skip[% END %][% IF rule.append %]Append[% ELSE %]Skip[% END %][% IF rule.remove %]Remove[% ELSE %]Skip[% END %][% IF rule.delete %]Delete[% ELSE %]Skip[% END %] + Delete + Edit + + +
    +
    + +
    + +
    + +
    +
    +
    +[% INCLUDE 'admin-menu.inc' %] +
    +
    +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-permissions.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-permissions.tt deleted file mode 100644 index 3189ebefb0..0000000000 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc-permissions.tt +++ /dev/null @@ -1,320 +0,0 @@ -[% USE Koha %] -[% INCLUDE 'doc-head-open.inc' %] -Koha › Administration › MARC field permissions -[% INCLUDE 'doc-head-close.inc' %] - -[% INCLUDE 'datatables.inc' %] - - - - - - -[% INCLUDE 'header.inc' %] -[% INCLUDE 'cat-search.inc' %] - - - -
    -
    -
    -
    - -

    Manage MARC field permissions

    - -[% UNLESS Koha.Preference( 'MARCPermissions' ) %] -
    - The MARCPermissions preference is not set, don't forget to enable it for rules to take effect. -
    -[% END %] -[% IF removeConfirm %] -
    -

    Remove rule?

    -

    Are you sure you want to remove the selected rule(s)?

    - -
    - -
    - -
    -[% END %] - -
    - - - - - - - - - - - - - - [% UNLESS edit %] - - - - - - - - - - - - - - - [% END %] - - [% FOREACH rule IN rules %] - - [% IF rule.edit %] - - - - - - - - - - - [% ELSE %] - - - - - - - - - - - [% END %] - - [% END %] - -
    RuleTagModuleFilterOn NewOn AppendedOn RemovedOn DeletedActions 
      - - - - - - - - - -
    [% rule.id %] - - - - - - - - - - - - - [% rule.id %][% rule.tagfield %][% rule.module %][% rule.filter %][% rule.on_new %][% rule.on_appended %][% rule.on_removed %][% rule.on_deleted %] - Delete - Edit - - [% IF rule.remove %] - - [% ELSE %] - - [% END %] -
    -
    - -
    - -
    - -
    -
    -
    -[% INCLUDE 'admin-menu.inc' %] -
    -
    -[% INCLUDE 'intranet-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref index f6da215317..6f2d8bd08e 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/cataloguing.pref @@ -278,8 +278,8 @@ Cataloging: - "Use of TY ( record type ) as a key will replace the default TY with the field value of your choosing." - - When importing records - - pref: MARCPermissions + - pref: MARCMergeRules choices: yes: "use" no: "don't use" - - MARC permissions rules to decide which action to take for each field. + - MARC merge rules to decide which action to take for each field. diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/logs.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/logs.pref index 0c2a1ffe8f..cb896c61a5 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/logs.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/logs.pref @@ -72,12 +72,6 @@ Logging: on: Log off: "Don't log" - when reports are added, deleted or changed. - - - - pref: MARCPermissionsLog - choices: - on: Log - off: "Don't log" - - MARC permissions rule evaluations when applying rules for MARC field modifications. Debugging: - - pref: DumpTemplateVarsIntranet diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-merge-rules.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-merge-rules.tt new file mode 100644 index 0000000000..49d3d63433 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-merge-rules.tt @@ -0,0 +1,144 @@ +[% INCLUDE 'help-top.inc' %] + +

    Manage MARC merge rules

    + +

    Rule evaluation

    +

    Rule evaluation begins with determining the active context. A context is a combination of a module and filter value. Depending on where in Koha marc merge rules are applied, different modules and filter values may be passed. + +

    A exact (non wildcard) filter match for an active module has the highest precedence. If there are multiple matches the context with the module with highest specificity is selected. If no exact filter matches are round, wildcard (*) matches are considered. A filter wildcard will match regardless of filter value. If multiple contexts matches, the context with highest module specificity is selected. If none are found we fallback to default actions (see below).

    + +

    Only rules for one context (module and filter combination) are ever selected. So if there is a set of rules with module "source" and filter "*", and another set of rules with module "source" and filter "bulkmarcimport". If the active context is module "source" and filter "bulkmarcimport" (as in bulkmarcimport.pl), only the rules for "source" and "bulkmarcimport" will be selected. The rules for module "source" filter "*" will NOT be included in rules selection.

    + +

    The selected rules are then applied. Rules are selected from most to least specific. This means that a more specific rule will override a less specific rule. * is less specific than a regular expression and regular expression is less specific than a normal field name (e.g. 245).

    + +

    Defaults

    +

    Default action when no matching rule is found is to overwrite field (on_new = add, on_appended = append, on_removed = remove, on_deleted = delete). If you wish to changed the default actions for fields and subfields, please add wildcard rules.

    + +

    Wildcards

    +

    * can be used as a wildcard for Tag and Filter. * is considered less specific than a non wildcard value, and thus will be overridden by a non wildcard value (e.g. "100", "a", etc).

    + +

    Regular expressions

    +

    Regular expressions can be used in the Tag field. Beware though, using regular expressions may create overlapping matches, in which case the first matching rule will be applied (in order of rule id).

    + +

    Actions

    + +

    In the following discussion "incoming record" refers to the new record to be merged with "old record" currently present in the Koha database. When merging fields Koha MARC merge rules are applied per field tag and differentiates between four different situations.

    + +

    For a specific field tag:

    + +

    A field is "added" (new) when old record has no fields with current tag.

    + +

    A field is "appended" when old record has fields with same tag but with different data.

    + +

    A field is "removed" when field is not present in incoming record but not all fields (with current tag) have been deleted.

    + +

    A field is "deleted" all fields with current tag has been deleted.

    + +

    Example rules

    +

    Following is matching conditions in order of specificity.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    TagModuleFilterDescription
    *[% modules.0.name %]*Match any field regardless of [% modules.0.name %]
    245[% modules.0.name %]*Match field 245 regardless of [% modules.0.name %]
    *[% modules.0.name %]z39.50Match any field when [% modules.0.name %] is z39.50
    5[0-9]{2}[% modules.0.name %]z39.50Match fields matching the regular expression 5[0-9]{2} (500-599) when [% modules.0.name %] is z39.50
    245[% modules.0.name %]z39.50Match field 245 when [% modules.0.name %] is z39.50
    + +
    + +

    Available filter modules

    +

    Filters cannot be regular expressions. Please use a single * as a wildcard if need.

    + + + + + + + + + [% FOREACH module IN modules %] + + + + + + [% END %] + +
    SpecificityModuleDescription
    [% module.specificity %][% module.name %][% module.description %]
    + +
    + +

    Available sources

    +

    The following sources currently implement MARCMergeRules.

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameDescription
    bulkmarcimportbin/migration_tools/bulkmarcimport.pl
    import_lexilebin/migration_tools/import_lexile.pl
    z39.50Import from Z39.50 search in browser
    intranetModifications from intranet in browser
    batchmodBatch record modification in browser
    batchimportBatch import of staged MARC records
    + +
    + +

    See the full documentation for Koha in the manual (online).

    + +[% INCLUDE 'help-bottom.inc' %] diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-permissions.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-permissions.tt deleted file mode 100644 index 9f657e659b..0000000000 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/help/marc-permissions.tt +++ /dev/null @@ -1,148 +0,0 @@ -[% INCLUDE 'help-top.inc' %] - -

    Manage MARC field permissions

    - -

    Rule evaluation

    -

    Rules are evaluated from least specific to most specific. This means that a more specific rule will override a less specific rule. * is less specific than a regular expression. A regular expression is less specific than a normal field name (e.g. 245). A Subfield tag is more specific than a Tag.

    -

    To add a field-specific rule (i.e. not a subfield), leave the Subfield tag field blank.

    - -

    Defaults

    -

    Default action when no rules exist is to replace the old record with the new record. Same as if MARCPermissions is disabled.

    -

    Default action when no matching rule is found is to leave a field unchanged (skip). If you wish to changed the default actions for fields and subfields, please add wildcard rules.

    - -

    Wildcards

    -

    * can be used as a wildcard for Tag, Subfield tag and Filter. * is considered less specific than a non wildcard value, and thus will be overridden by a non wildcard value (e.g. "100", "a", etc).

    - -

    Regular expressions

    -

    Regular expressions can be used in both the Tag and Subfield tag fields. Beware though, using regular expressions may create overlapping matches, in which case they will be applied in a sorted order.

    - -

    Example rules

    -

    Following is an example rule set in order of specificity.

    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    TagSubfield tagModuleFilterDescription
    *[% modules.0.name %]*Match any field regardless of [% modules.0.name %]
    **[% modules.0.name %]*Match any subfield regardless of [% modules.0.name %]
    245*[% modules.0.name %]z39.50Match any subfield under 245 when [% modules.0.name %] is z39.50
    *b[% modules.0.name %]z39.50Match subfield b regardless of field when [% modules.0.name %] is z39.50
    500[a-d][% modules.0.name %]z39.50Match subfields a, b, c, d when field is 500 and [% modules.0.name %] is z39.50
    5..a[% modules.0.name %]z39.50Match subfield a when field matches the regular expression 5.. (500-599) and [% modules.0.name %] is z39.50
    245a[% modules.0.name %]z39.50Match subfield a when field is 500 and [% modules.0.name %] is z39.50
    - -
    - -

    Available filter modules

    -

    Filters cannot be regular expressions. Please use a single * as a wildcard if need.

    - - - - - - - - - [% FOREACH module IN modules %] - - - - - - [% END %] - -
    SpecificityModuleDescription
    [% module.specificity %][% module.name %][% module.description %]
    - -
    - -

    Available sources

    -

    The following sources currently implement MARCPermissions.

    - - - - - - - - - - - - - - - - - - - - - - - - - - -
    NameDescription
    bulkmarcimportbin/migration_tools/bulkmarcimport.pl
    import_lexilebin/migration_tools/import_lexile.pl
    z39.50Import from Z39.50 search in browser
    intranetModifications from intranet in browser
    batchmodBatch record modification in browser
    - -
    - -

    Indicators

    -

    Indicators can be addressed as i1 and i2 in the Subfield tag field. Some actions might not be available for indicators, in which case they will be disabled.

    - -
    - -

    Logging

    -

    If MARCPermissionsLog is enabled, log entries for each record modification will be available in the Modification log in the Catalog module under the Modify action. This can be very helpful when debugging rule sets.

    -

    The administration area is where you set all of your preferences for the system. Preference are broken down into several categories, detailed below.

    - -

    See the full documentation for Koha in the manual (online).

    - -[% INCLUDE 'help-bottom.inc' %] diff --git a/t/db_dependent/Biblio/MARCPermissions.t b/t/db_dependent/Biblio/MARCPermissions.t deleted file mode 100755 index 1b6d26f74d..0000000000 --- a/t/db_dependent/Biblio/MARCPermissions.t +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/perl - -# This file is part of Koha. -# -# Koha is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# Koha is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Koha; if not, see . - -use Modern::Perl; - -use Test::More tests => 16; -use Test::MockModule; - -use MARC::Record; -use Data::Dumper; - -BEGIN { - use_ok('C4::Biblio'); -} - -# Start transaction -my $dbh = C4::Context->dbh; -$dbh->{AutoCommit} = 0; -$dbh->{RaiseError} = 1; - -C4::Context->set_preference( 'MARCPermissions', '1'); -C4::Context->set_preference( 'MARCPermissionsLog', '1'); - -# Create a record -my $record = MARC::Record->new(); -$record->append_fields ( - MARC::Field->new('008', '12345'), - MARC::Field->new('100', '','', 'a' => 'field data for 100 a without indicators'), - MARC::Field->new('250', '','', 'a' => '250 bottles of beer on the wall'), - MARC::Field->new('245', '1','2', 'a' => 'field data for 245 a with indicators 12'), - MARC::Field->new('500', '1','1', 'a' => 'the lazy programmer jumps over the quick brown tests'), - MARC::Field->new('500', '2','2', 'a' => 'the quick brown test jumps over the lazy programmers'), -); - -# Add record to DB -my ($biblionumber, $biblioitemnumber) = AddBiblio($record, ''); - -my $modules = GetMarcPermissionsModules(); - -############################################################################## -# Test overwrite rule -my $mod_record = MARC::Record->new(); -$mod_record->append_fields ( - MARC::Field->new('008', '12345'), - MARC::Field->new('100', '','', 'a' => 'field data for 100 a without indicators'), - MARC::Field->new('245', '1','2', 'a' => 'field data for 245 a with indicators 12'), - MARC::Field->new('500', '1','1', 'a' => 'this field has now been changed'), - MARC::Field->new('500', '2','2', 'a' => 'and so have this field'), -); - -# Clear MARC permission rules from DB -DelMarcPermissionsRule($_->{id}) for GetMarcPermissionsRules(); - -# Add MARC permission rules to DB -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '*', - tagsubfield => '', - filter => '*', - on_existing => 'overwrite', - on_new => 'add', - on_removed => 'remove' -}); - -my @log = (); -my $new_record = ApplyMarcPermissions({ - biblionumber => $biblionumber, - record => $mod_record, - frameworkcode => '', - filter => {$modules->[0]->{'name'} => 'foo'}, - log => \@log - }); - -my @a500 = $new_record->field('500'); -is ($a500[0]->subfield('a'), 'this field has now been changed', 'old field is replaced when overwrite'); -is ($a500[1]->subfield('a'), 'and so have this field', 'old field is replaced when overwrite'); - -############################################################################## -# Test remove rule -is ($new_record->field('250'), undef, 'removed field is removed'); - -############################################################################## -# Test skip rule -$mod_record = MARC::Record->new(); -$mod_record->append_fields ( - MARC::Field->new('008', '12345'), - MARC::Field->new('100', '','', 'a' => 'field data for 100 a without indicators'), - MARC::Field->new('245', '1','2', 'a' => 'field data for 245 a with indicators 12'), - MARC::Field->new('500', '1','1', 'a' => 'this should not show'), - MARC::Field->new('500', '2','2', 'a' => 'and neither should this'), -); - -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '500', - tagsubfield => '*', - filter => '*', - on_existing => 'skip', - on_new => 'skip', - on_removed => 'skip' -}); - -@log = (); -$new_record = ApplyMarcPermissions({ - biblionumber => $biblionumber, - record => $mod_record, - frameworkcode => '', - filter => {$modules->[0]->{'name'} => 'foo'}, - log => \@log - }); - -@a500 = $new_record->field('500'); -is ($a500[0]->subfield('a'), 'the lazy programmer jumps over the quick brown tests', 'old field is kept when skip'); -is ($a500[1]->subfield('a'), 'the quick brown test jumps over the lazy programmers', 'old field is kept when skip'); - -############################################################################## -# Test add rule -$mod_record = MARC::Record->new(); -$mod_record->append_fields ( - MARC::Field->new('008', '12345'), - MARC::Field->new('100', '','', 'a' => 'field data for 100 a without indicators'), - MARC::Field->new('250', '','', 'a' => '250 bottles of beer on the wall'), - #MARC::Field->new('245', '1','2', 'a' => 'field data for 245 a with indicators 12'), - MARC::Field->new('245', '1','2', 'a' => 'some new fun value'), - MARC::Field->new('500', '1','1', 'a' => 'the lazy programmer jumps over the quick brown tests'), - MARC::Field->new('500', '2','2', 'a' => 'the quick brown test jumps over the lazy programmers'), -); - -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '245', - tagsubfield => '*', - filter => '*', - on_existing => 'add', - on_new => 'add', - on_removed => 'skip' -}); - - -@log = (); -$new_record = ApplyMarcPermissions({ - biblionumber => $biblionumber, - record => $mod_record, - frameworkcode => '', - filter => {$modules->[0]->{'name'} => 'foo'}, - log => \@log - }); - -my @a245 = $new_record->field('245')->subfield('a'); -is ($a245[0], 'field data for 245 a with indicators 12', 'old field is kept when adding new'); -is ($a245[1], 'some new fun value', 'new field is added'); - -############################################################################## -# Test add_or_correct rule -$mod_record = MARC::Record->new(); -$mod_record->append_fields ( - MARC::Field->new('008', '12345'), - #MARC::Field->new('100', '','', 'a' => 'field data for 100 a without indicators'), - MARC::Field->new('100', '','', 'a' => 'a very different value'), - MARC::Field->new('250', '','', 'a' => '250 bottles of beer on the wall'), - #MARC::Field->new('245', '1','2', 'a' => 'field data for 245 a with indicators 12'), - MARC::Field->new('245', '1','2', 'a' => 'Field data for 245 a with indicators 12', 'a' => 'some very different value'), - MARC::Field->new('500', '1','1', 'a' => 'the lazy programmer jumps over the quick brown tests'), - MARC::Field->new('500', '2','2', 'a' => 'the quick brown test jumps over the lazy programmers'), -); - -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '(100|245)', - tagsubfield => '*', - filter => '*', - on_existing => 'add_or_correct', - on_new => 'add', - on_removed => 'skip' -}); - -@log = (); -$new_record = ApplyMarcPermissions({ - biblionumber => $biblionumber, - record => $mod_record, - frameworkcode => '', - filter => {$modules->[0]->{'name'} => 'foo'}, - log => \@log - }); - -@a245 = $new_record->field('245')->subfield('a'); -is ($a245[0], 'Field data for 245 a with indicators 12', 'add_or_correct modifies field when a correction'); -is ($a245[1], 'some very different value', 'add_or_correct adds field when not a correction'); - -my @a100 = $new_record->field('100')->subfield('a'); -is ($a100[0], 'field data for 100 a without indicators', 'add_or_correct keeps old field when not a correction'); -is ($a100[1], 'a very different value', 'add_or_correct adds field when not a correction'); - -############################################################################## -# Test rule evaluation order -$mod_record = MARC::Record->new(); -$mod_record->append_fields ( - MARC::Field->new('008', '12345'), - MARC::Field->new('100', '','', 'a' => 'field data for 100 a without indicators'), - MARC::Field->new('250', '','', 'a' => 'take one down, pass it around'), - MARC::Field->new('245', '1','2', 'a' => 'field data for 245 a with indicators 12'), - MARC::Field->new('500', '1','1', 'a' => 'the lazy programmer jumps over the quick brown tests'), - MARC::Field->new('500', '2','2', 'a' => 'the quick brown test jumps over the lazy programmers'), -); - - -DelMarcPermissionsRule($_->{id}) for GetMarcPermissionsRules(); - -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '*', - tagsubfield => '*', - filter => '*', - on_existing => 'skip', - on_new => 'skip', - on_removed => 'skip' -}); -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '250', - tagsubfield => '*', - filter => '*', - on_existing => 'overwrite', - on_new => 'skip', - on_removed => 'skip' -}); -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '*', - tagsubfield => 'a', - filter => '*', - on_existing => 'add_or_correct', - on_new => 'skip', - on_removed => 'skip' -}); -AddMarcPermissionsRule({ - module => $modules->[0]->{'id'}, - tagfield => '250', - tagsubfield => 'a', - filter => '*', - on_existing => 'add', - on_new => 'skip', - on_removed => 'skip' -}); - -@log = (); -$new_record = ApplyMarcPermissions({ - biblionumber => $biblionumber, - record => $mod_record, - frameworkcode => '', - filter => {$modules->[0]->{'name'} => 'foo'}, - log => \@log - }); - -my @rule = grep { $_->{tag} eq '250' and $_->{subfieldcode} eq 'a' } @log; -is(scalar @rule, 1, 'only one rule applied'); -is($rule[0]->{event}.':'.$rule[0]->{action}, 'existing:add', 'most specific rule used'); - -my @a250 = $new_record->field('250')->subfield('a'); -is ($a250[0], '250 bottles of beer on the wall', 'most specific rule is applied, original field kept'); -is ($a250[1], 'take one down, pass it around', 'most specific rule is applied, new field added'); - -$dbh->rollback; - -1; diff --git a/t/db_dependent/Biblio/MarcMergeRules.t b/t/db_dependent/Biblio/MarcMergeRules.t new file mode 100755 index 0000000000..8dd6aa8ae1 --- /dev/null +++ b/t/db_dependent/Biblio/MarcMergeRules.t @@ -0,0 +1,689 @@ +#!/usr/bin/perl + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use MARC::Record; + +use C4::Context; +use C4::Biblio; +use Koha::Database; #?? + +use Test::More tests => 20; +use Test::MockModule; + +use Koha::MarcMergeRules; +use Koha::MarcMergeRulesModules; + +use t::lib::Mocks; + +my $schema = Koha::Database->schema; +$schema->storage->txn_begin; + +t::lib::Mocks::mock_preference('MARCMergeRules', '1'); + +# Create a record +my $orig_record = MARC::Record->new(); +$orig_record->append_fields ( + MARC::Field->new('250', '','', 'a' => '250 bottles of beer on the wall'), + MARC::Field->new('250', '','', 'a' => '256 bottles of beer on the wall'), + MARC::Field->new('500', '','', 'a' => 'One bottle of beer in the fridge'), +); + +# Order modules by specificity and get first two +my $modules_rs = Koha::MarcMergeRulesModules->search(undef, { order_by => { -desc => 'specificity' } }); +my @modules; +push @modules, $modules_rs->next; +push @modules, $modules_rs->next; + + +my $incoming_record = MARC::Record->new(); +$incoming_record->append_fields( + MARC::Field->new('250', '', '', 'a' => '256 bottles of beer on the wall'), # Unchanged + MARC::Field->new('250', '', '', 'a' => '251 bottles of beer on the wall'), # Appended + # MARC::Field->new('250', '', '', 'a' => '250 bottles of beer on the wall'), # Removed + # MARC::Field->new('500', '', '', 'a' => 'One bottle of beer in the fridge'), # Deleted + MARC::Field->new('501', '', '', 'a' => 'One cold bottle of beer in the fridge'), # Added + MARC::Field->new('501', '', '', 'a' => 'Two cold bottles of beer in the fridge'), # Added +); + +# Test default behavior when MARCMergeRules is enabled, but no rules defined (overwrite) +subtest 'Record fields has been overwritten when no merge rules are defined' => sub { + plan tests => 4; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +my $rule = Koha::MarcMergeRules->find_or_create({ + tag => '*', + module => $modules[0]->id, + filter => '*', + add => 0, + append => 0, + remove => 0, + delete => 0 +}); + +subtest 'Record fields has been protected when matched merge all rule operations are set to "0"' => sub { + plan tests => 3; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + + +subtest 'Only new fields has been added when add = 1, append = 0, remove = 0, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields retain their original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field retain it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Only appended fields has been added when add = 0, append = 1, remove = 0, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"251" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + +}; + +subtest 'Appended and added fields has been added when add = 1, append = 1, remove = 0, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 0, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 6, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"251" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been only removed when add = 0, append = 0, remove = 1, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Record fields has been added and removed when add = 1, append = 0, remove = 1, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended and removed when add = 0, append = 1, remove = 1, delete = 0' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +subtest 'Record fields has been added, appended and removed when add = 0, append = 1, remove = 1, delete = 0' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 1, + 'delete' => 0, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been deleted when add = 0, append = 0, remove = 0, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); +}; + +subtest 'Record fields has been added and deleted when add = 1, append = 0, remove = 0, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall'], + '"250" fields has retained their original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended and deleted when add = 0, append = 1, remove = 0, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" field has been appended' + ); +}; + +subtest 'Record fields has been added, appended and deleted when add = 1, append = 1, remove = 0, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 0, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['250 bottles of beer on the wall', '256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" field has been appended' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been removed and deleted when add = 0, append = 0, remove = 1, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 0, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 1, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); +}; + +subtest 'Record fields has been added, removed and deleted when add = 1, append = 0, remove = 1, delete = 1' => sub { + plan tests => 3; + + $rule->set( + { + 'add' => 1, + 'append' => 0, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall'], + '"250" field has been removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +subtest 'Record fields has been appended, removed and deleted when add = 0, append = 1, remove = 1, delete = 1' => sub { + plan tests => 2; + + $rule->set( + { + 'add' => 0, + 'append' => 1, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + cmp_ok(scalar @all_fields, '==', 2, "Record has the expected number of fields"); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); +}; + +subtest 'Record fields has been overwritten when add = 1, append = 1, remove = 1, delete = 1' => sub { + plan tests => 4; + + $rule->set( + { + 'add' => 1, + 'append' => 1, + 'remove' => 1, + 'delete' => 1, + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +# Test rule tag specificity + +# Protect field 500 with more specific tag value +my $skip_all_rule = Koha::MarcMergeRules->find_or_create({ + tag => '500', + module => $modules[0]->id, + filter => '*', + add => 0, + append => 0, + remove => 0, + delete => 0 +}); + +subtest '"500" field has been protected when rule matching on tag "500" is add = 0, append = 0, remove = 0, delete = 0' => sub { + plan tests => 4; + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 5, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +# Test regexp matching +subtest '"5XX" fields has been protected when rule matching on regexp "5\d{2}" is add = 0, append = 0, remove = 0, delete = 0' => sub { + plan tests => 3; + + $skip_all_rule->set( + { + 'tag' => '5\d{2}', + } + ); + $skip_all_rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 3, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('500') ], + ['One bottle of beer in the fridge'], + '"500" field has retained it\'s original value' + ); +}; + +# Test module specificity, the 0 all rule should no longer be included in set of applied rules +subtest 'Record fields has been overwritten when non wild card rule with filter match is add = 1, append = 1, remove = 1, delete = 1' => sub { + plan tests => 4; + + $rule->set( + { + 'filter' => 'test', + } + ); + $rule->store(); + + my $merged_record = Koha::MarcMergeRules->merge_records($orig_record, $incoming_record, { $modules[0]->name => 'test' }); + + my @all_fields = $merged_record->fields(); + + cmp_ok(scalar @all_fields, '==', 4, "Record has the expected number of fields"); + is_deeply( + [map { $_->subfield('a') } $merged_record->field('250') ], + ['256 bottles of beer on the wall', '251 bottles of beer on the wall'], + '"250" fields has been appended and removed' + ); + + my @fields = $merged_record->field('500'); + cmp_ok(scalar @fields, '==', 0, '"500" field has been deleted'); + + is_deeply( + [map { $_->subfield('a') } $merged_record->field('501') ], + ['One cold bottle of beer in the fridge', 'Two cold bottles of beer in the fridge'], + '"501" fields has been added' + ); +}; + +$schema->storage->txn_rollback; + +1; -- 2.11.0