From 34deeb51cc48b9da5f86d52f1635c0810aa6e9ae Mon Sep 17 00:00:00 2001 From: Hammat Wele Date: Wed, 28 Aug 2024 21:39:58 +0000 Subject: [PATCH] Bug 36027: (follow-up) Tidy search_for_data_inconsistencies.pl --- .../search_for_data_inconsistencies.pl | 439 ++++++++++-------- 1 file changed, 234 insertions(+), 205 deletions(-) diff --git a/misc/maintenance/search_for_data_inconsistencies.pl b/misc/maintenance/search_for_data_inconsistencies.pl index ddb86758fd..aa12a5220b 100755 --- a/misc/maintenance/search_for_data_inconsistencies.pl +++ b/misc/maintenance/search_for_data_inconsistencies.pl @@ -16,6 +16,7 @@ # along with Koha; if not, see . use Modern::Perl; + use Koha::Script; use Koha::AuthorisedValues; use Koha::Authorities; @@ -80,8 +81,10 @@ pod2usage(1) if $options{'help'}; # Set skip options based on check options set_skip_options(); + # Set all check options to 1 if none are provided, unless specified skip options set_all_check_options_if_none_provided(); + # Run checks based on options CheckItemsBranch() if $options{'check-branch'} && !$options{'skip-branch'}; CheckItemsAuthHeader() if $options{'check-auth'} && !$options{'skip-auth'}; @@ -100,17 +103,20 @@ sub handle_invalid_options { # Set skip options based on check options sub set_skip_options { + # If at least one check option is provided, set all skip options to 0 my $has_check_option = grep { $options{$_} } grep { /^check-/ } keys %options; if ($has_check_option) { + # if a least one skip option is provided, print a warning my $has_skip_option = grep { $options{$_} == 1 } grep { /^skip-/ } keys %options; if ($has_skip_option) { - print("Warning : skip options are ignored when check options are provided\n") + print("Warning : skip options are ignored when check options are provided\n"); } + # Set all skip options to 0 - foreach my $key (keys %options) { - if ($key =~ /^skip-/) { + foreach my $key ( keys %options ) { + if ( $key =~ /^skip-/ ) { $options{$key} = 0; } } @@ -121,11 +127,12 @@ sub set_skip_options { # Set all check options to 1 if none are provided, considering skip options sub set_all_check_options_if_none_provided { my $any_check_option_provided = grep { $options{$_} } grep { /^check-/ } keys %options; - if (!$any_check_option_provided) { - foreach my $key (keys %options) { - if ($key =~ /^check-/) { + if ( !$any_check_option_provided ) { + foreach my $key ( keys %options ) { + if ( $key =~ /^check-/ ) { my $skip_key = $key; $skip_key =~ s/^check-/skip-/; + # Set check option to 1 unless the corresponding skip option indicated $options{$key} = 1 unless $options{$skip_key}; } @@ -134,261 +141,283 @@ sub set_all_check_options_if_none_provided { } sub CheckItemsBranch { - my $items = Koha::Items->search({ -or => { homebranch => undef, holdingbranch => undef }}); - if ( $items->count ) { new_section("Not defined items.homebranch and/or items.holdingbranch")} + my $items = Koha::Items->search( { -or => { homebranch => undef, holdingbranch => undef } } ); + if ( $items->count ) { new_section("Not defined items.homebranch and/or items.holdingbranch") } while ( my $item = $items->next ) { if ( not $item->homebranch and not $item->holdingbranch ) { - new_item(sprintf "Item with itemnumber=%s and biblionumber=%s does not have homebranch and holdingbranch defined", $item->itemnumber, $item->biblionumber); - } elsif ( not $item->homebranch ) { - new_item(sprintf "Item with itemnumber=%s and biblionumber=%s does not have homebranch defined", $item->itemnumber, $item->biblionumber); - } else { - new_item(sprintf "Item with itemnumber=%s and biblionumber=%s does not have holdingbranch defined", $item->itemnumber, $item->biblionumber); + new_item( + sprintf + "Item with itemnumber=%s and biblionumber=%s does not have homebranch and holdingbranch defined", + $item->itemnumber, $item->biblionumber + ); + } elsif ( not $item->homebranch ) { + new_item( + sprintf "Item with itemnumber=%s and biblionumber=%s does not have homebranch defined", + $item->itemnumber, $item->biblionumber + ); + } else { + new_item( + sprintf "Item with itemnumber=%s and biblionumber=%s does not have holdingbranch defined", + $item->itemnumber, $item->biblionumber + ); } } - if ( $items->count ) { new_hint("Edit these items and set valid homebranch and/or holdingbranch")} + if ( $items->count ) { new_hint("Edit these items and set valid homebranch and/or holdingbranch") } } sub CheckItemsAuthHeader { + # No join possible, FK is missing at DB level - my @auth_types = Koha::Authority::Types->search->get_column('authtypecode'); - my $authorities = Koha::Authorities->search({authtypecode => { 'not in' => \@auth_types } }); - if ( $authorities->count ) {new_section("Invalid auth_header.authtypecode")} + my @auth_types = Koha::Authority::Types->search->get_column('authtypecode'); + my $authorities = Koha::Authorities->search( { authtypecode => { 'not in' => \@auth_types } } ); + if ( $authorities->count ) { new_section("Invalid auth_header.authtypecode") } while ( my $authority = $authorities->next ) { - new_item(sprintf "Authority with authid=%s does not have a code defined (%s)", $authority->authid, $authority->authtypecode); + new_item( + sprintf "Authority with authid=%s does not have a code defined (%s)", $authority->authid, + $authority->authtypecode + ); } - if ( $authorities->count ) {new_hint("Go to 'Home -> Administration -> Authority types' to define them")}; - }; - - sub CheckItemsStatus { - if ( C4::Context->preference('item-level_itypes') ) { - my $items_without_itype = Koha::Items->search( { -or => [itype => undef,itype => ''] } ); - if ( $items_without_itype->count ) { - new_section("Items do not have itype defined"); - while ( my $item = $items_without_itype->next ) { - if (defined $item->biblioitem->itemtype && $item->biblioitem->itemtype ne '' ) { - new_item( - sprintf "Item with itemnumber=%s and biblionumber = %s does not have a itype value, biblio's item type will be used (%s)", + if ( $authorities->count ) { new_hint("Go to 'Home -> Administration -> Authority types' to define them") } +} + +sub CheckItemsStatus { + if ( C4::Context->preference('item-level_itypes') ) { + my $items_without_itype = Koha::Items->search( { -or => [ itype => undef, itype => '' ] } ); + if ( $items_without_itype->count ) { + new_section("Items do not have itype defined"); + while ( my $item = $items_without_itype->next ) { + if ( defined $item->biblioitem->itemtype && $item->biblioitem->itemtype ne '' ) { + new_item( + sprintf + "Item with itemnumber=%s and biblionumber = %s does not have a itype value, biblio's item type will be used (%s)", $item->itemnumber, $item->biblioitem->biblionumber, $item->biblioitem->itemtype - ); - } else { - new_item( - sprintf "Item with itemnumber=%s and biblionumber = %s does not have a itype value, additionally no item type defined for biblionumber=%s", + ); + } else { + new_item( + sprintf + "Item with itemnumber=%s and biblionumber = %s does not have a itype value, additionally no item type defined for biblionumber=%s", $item->itemnumber, $item->biblioitem->biblionumber, $item->biblioitem->biblionumber - ); - } + ); } - new_hint("The system preference item-level_itypes expects item types to be defined at item level"); } - }else{ - my $biblioitems_without_itemtype = Koha::Biblioitems->search( { itemtype => undef } ); - if ( $biblioitems_without_itemtype->count ) { - new_section("Biblioitems do not have itemtype defined"); - while ( my $biblioitem = $biblioitems_without_itemtype->next ) { - new_item( + new_hint("The system preference item-level_itypes expects item types to be defined at item level"); + } + } else { + my $biblioitems_without_itemtype = Koha::Biblioitems->search( { itemtype => undef } ); + if ( $biblioitems_without_itemtype->count ) { + new_section("Biblioitems do not have itemtype defined"); + while ( my $biblioitem = $biblioitems_without_itemtype->next ) { + new_item( sprintf "Biblioitem with biblioitemnumber=%s does not have a itemtype value", $biblioitem->biblioitemnumber - ); - } - new_hint("The system preference item-level_itypes expects item types to be defined at biblio level"); + ); } + new_hint("The system preference item-level_itypes expects item types to be defined at biblio level"); } + } - my @itemtypes = Koha::ItemTypes->search->get_column('itemtype'); - if ( C4::Context->preference('item-level_itypes') ) { - my $items_with_invalid_itype = Koha::Items->search( { -and => [itype => { not_in => \@itemtypes }, itype => { '!=' => '' }] } ); - if ( $items_with_invalid_itype->count ) { - new_section("Items have invalid itype defined"); - while ( my $item = $items_with_invalid_itype->next ) { - new_item( + my @itemtypes = Koha::ItemTypes->search->get_column('itemtype'); + if ( C4::Context->preference('item-level_itypes') ) { + my $items_with_invalid_itype = + Koha::Items->search( { -and => [ itype => { not_in => \@itemtypes }, itype => { '!=' => '' } ] } ); + if ( $items_with_invalid_itype->count ) { + new_section("Items have invalid itype defined"); + while ( my $item = $items_with_invalid_itype->next ) { + new_item( sprintf "Item with itemnumber=%s, biblionumber=%s does not have a valid itype value (%s)", $item->itemnumber, $item->biblionumber, $item->itype - ); - } - new_hint("The items must have a itype value that is defined in the item types of Koha (Home › Administration › Item types administration)"); + ); } - }else{ - my $biblioitems_with_invalid_itemtype = Koha::Biblioitems->search( - { itemtype => { not_in => \@itemtypes } } + new_hint( + "The items must have a itype value that is defined in the item types of Koha (Home › Administration › Item types administration)" ); - if ( $biblioitems_with_invalid_itemtype->count ) { - new_section("Biblioitems do not have itemtype defined"); - while ( my $biblioitem = $biblioitems_with_invalid_itemtype->next ) { - new_item( + } + } else { + my $biblioitems_with_invalid_itemtype = Koha::Biblioitems->search( { itemtype => { not_in => \@itemtypes } } ); + if ( $biblioitems_with_invalid_itemtype->count ) { + new_section("Biblioitems do not have itemtype defined"); + while ( my $biblioitem = $biblioitems_with_invalid_itemtype->next ) { + new_item( sprintf "Biblioitem with biblioitemnumber=%s does not have a valid itemtype value", $biblioitem->biblioitemnumber - ); - } - new_hint("The biblioitems must have a itemtype value that is defined in the item types of Koha (Home › Administration › Item types administration)"); + ); } + new_hint( + "The biblioitems must have a itemtype value that is defined in the item types of Koha (Home › Administration › Item types administration)" + ); } + } - my ( @decoding_errors, @ids_not_in_marc ); - my $biblios = Koha::Biblios->search; - my ( $biblio_tag, $biblio_subfield ) = C4::Biblio::GetMarcFromKohaField( "biblio.biblionumber" ); - my ( $biblioitem_tag, $biblioitem_subfield ) = C4::Biblio::GetMarcFromKohaField( "biblioitems.biblioitemnumber" ); - while ( my $biblio = $biblios->next ) { - my $record = eval{$biblio->metadata->record;}; - if ($@) { - push @decoding_errors, $@; - next; - } - my ( $biblionumber, $biblioitemnumber ); - if ( $biblio_tag < 10 ) { - my $biblio_control_field = $record->field($biblio_tag); - $biblionumber = $biblio_control_field->data if $biblio_control_field; - } else { - $biblionumber = $record->subfield( $biblio_tag, $biblio_subfield ); - } - if ( $biblioitem_tag < 10 ) { - my $biblioitem_control_field = $record->field($biblioitem_tag); - $biblioitemnumber = $biblioitem_control_field->data if $biblioitem_control_field; - } else { - $biblioitemnumber = $record->subfield( $biblioitem_tag, $biblioitem_subfield ); - } - if ( $biblionumber != $biblio->biblionumber ) { - push @ids_not_in_marc, + my ( @decoding_errors, @ids_not_in_marc ); + my $biblios = Koha::Biblios->search; + my ( $biblio_tag, $biblio_subfield ) = C4::Biblio::GetMarcFromKohaField("biblio.biblionumber"); + my ( $biblioitem_tag, $biblioitem_subfield ) = C4::Biblio::GetMarcFromKohaField("biblioitems.biblioitemnumber"); + while ( my $biblio = $biblios->next ) { + my $record = eval { $biblio->metadata->record; }; + if ($@) { + push @decoding_errors, $@; + next; + } + my ( $biblionumber, $biblioitemnumber ); + if ( $biblio_tag < 10 ) { + my $biblio_control_field = $record->field($biblio_tag); + $biblionumber = $biblio_control_field->data if $biblio_control_field; + } else { + $biblionumber = $record->subfield( $biblio_tag, $biblio_subfield ); + } + if ( $biblioitem_tag < 10 ) { + my $biblioitem_control_field = $record->field($biblioitem_tag); + $biblioitemnumber = $biblioitem_control_field->data if $biblioitem_control_field; + } else { + $biblioitemnumber = $record->subfield( $biblioitem_tag, $biblioitem_subfield ); + } + if ( $biblionumber != $biblio->biblionumber ) { + push @ids_not_in_marc, { biblionumber => $biblio->biblionumber, biblionumber_in_marc => $biblionumber, }; - } - if ( $biblioitemnumber != $biblio->biblioitem->biblioitemnumber ) { - push @ids_not_in_marc, + } + if ( $biblioitemnumber != $biblio->biblioitem->biblioitemnumber ) { + push @ids_not_in_marc, { - biblionumber => $biblio->biblionumber, - biblioitemnumber => $biblio->biblioitem->biblioitemnumber, + biblionumber => $biblio->biblionumber, + biblioitemnumber => $biblio->biblioitem->biblioitemnumber, biblioitemnumber_in_marc => $biblionumber, }; - } - } - if ( @decoding_errors ) { - new_section("Bibliographic records have invalid MARCXML"); - new_item($_) for @decoding_errors; - new_hint("The bibliographic records must have a valid MARCXML or you will face encoding issues or wrong displays"); } - if (@ids_not_in_marc) { - new_section("Bibliographic records have MARCXML without biblionumber or biblioitemnumber"); - for my $id (@ids_not_in_marc) { - if ( exists $id->{biblioitemnumber} ) { - new_item( - sprintf(q{Biblionumber %s has biblioitemnumber '%s' but should be '%s' in %s$%s}, - $id->{biblionumber}, - $id->{biblioitemnumber}, - $id->{biblioitemnumber_in_marc}, - $biblioitem_tag, - $biblioitem_subfield, + } + if (@decoding_errors) { + new_section("Bibliographic records have invalid MARCXML"); + new_item($_) for @decoding_errors; + new_hint( + "The bibliographic records must have a valid MARCXML or you will face encoding issues or wrong displays"); + } + if (@ids_not_in_marc) { + new_section("Bibliographic records have MARCXML without biblionumber or biblioitemnumber"); + for my $id (@ids_not_in_marc) { + if ( exists $id->{biblioitemnumber} ) { + new_item( + sprintf( + q{Biblionumber %s has biblioitemnumber '%s' but should be '%s' in %s$%s}, + $id->{biblionumber}, + $id->{biblioitemnumber}, + $id->{biblioitemnumber_in_marc}, + $biblioitem_tag, + $biblioitem_subfield, ) - ); - }else{ - new_item( - sprintf(q{Biblionumber %s has '%s' in %s$%s}, - $id->{biblionumber}, - $id->{biblionumber_in_marc}, - $biblio_tag, - $biblio_subfield, + ); + } else { + new_item( + sprintf( + q{Biblionumber %s has '%s' in %s$%s}, + $id->{biblionumber}, + $id->{biblionumber_in_marc}, + $biblio_tag, + $biblio_subfield, ) - ); - } + ); } - new_hint("The bibliographic records must have the biblionumber and biblioitemnumber in MARCXML"); } + new_hint("The bibliographic records must have the biblionumber and biblioitemnumber in MARCXML"); } +} - sub CheckItemsFramework { - my @framework_codes = Koha::BiblioFrameworks->search()->get_column('frameworkcode'); - push @framework_codes,""; # The default is not stored in frameworks, we need to force it - - my $invalid_av_per_framework = {}; - foreach my $frameworkcode ( @framework_codes ) { - # We are only checking fields that are mapped to DB fields - my $msss = Koha::MarcSubfieldStructures->search({ - frameworkcode => $frameworkcode, - authorised_value => { - '!=' => [ -and => ( undef, '' ) ] - }, - kohafield => { - '!=' => [ -and => ( undef, '' ) ] - } - }); +sub CheckItemsFramework { + my @framework_codes = Koha::BiblioFrameworks->search()->get_column('frameworkcode'); + push @framework_codes, ""; # The default is not stored in frameworks, we need to force it + + my $invalid_av_per_framework = {}; + foreach my $frameworkcode (@framework_codes) { + + # We are only checking fields that are mapped to DB fields + my $msss = Koha::MarcSubfieldStructures->search( + { + frameworkcode => $frameworkcode, + authorised_value => { '!=' => [ -and => ( undef, '' ) ] }, + kohafield => { '!=' => [ -and => ( undef, '' ) ] } + } + ); while ( my $mss = $msss->next ) { my $kohafield = $mss->kohafield; - my $av = $mss->authorised_value; - next if grep {$_ eq $av} qw( branches itemtypes cn_source ); # internal categories + my $av = $mss->authorised_value; + next if grep { $_ eq $av } qw( branches itemtypes cn_source ); # internal categories my $avs = Koha::AuthorisedValues->search_by_koha_field( - { - frameworkcode => $frameworkcode, - kohafield => $kohafield, - } - ); - my $tmp_kohafield = $kohafield; - if ( $tmp_kohafield =~ /^biblioitems/ ) { - $tmp_kohafield =~ s|biblioitems|biblioitem|; + { + frameworkcode => $frameworkcode, + kohafield => $kohafield, + } + ); + my $tmp_kohafield = $kohafield; + if ( $tmp_kohafield =~ /^biblioitems/ ) { + $tmp_kohafield =~ s|biblioitems|biblioitem|; } else { - $tmp_kohafield =~ s|items|me|; - } - # replace items.attr with me.attr + $tmp_kohafield =~ s|items|me|; + } - # We are only checking biblios with items - my $items = Koha::Items->search( - { - $tmp_kohafield => - { - -not_in => [ $avs->get_column('authorised_value'), '' ], - '!=' => undef, - }, - 'biblio.frameworkcode' => $frameworkcode - }, - { join => [ 'biblioitem', 'biblio' ] } - ); - if ( $items->count ) { - $invalid_av_per_framework->{ $frameworkcode }->{$av} = - { items => $items, kohafield => $kohafield }; + # replace items.attr with me.attr + + # We are only checking biblios with items + my $items = Koha::Items->search( + { + $tmp_kohafield => { + -not_in => [ $avs->get_column('authorised_value'), '' ], + '!=' => undef, + }, + 'biblio.frameworkcode' => $frameworkcode + }, + { join => [ 'biblioitem', 'biblio' ] } + ); + if ( $items->count ) { + $invalid_av_per_framework->{$frameworkcode}->{$av} = + { items => $items, kohafield => $kohafield }; + } } } -} -if (%$invalid_av_per_framework) { - new_section('Wrong values linked to authorised values'); - for my $frameworkcode ( keys %$invalid_av_per_framework ) { - while ( my ( $av_category, $v ) = each %{$invalid_av_per_framework->{$frameworkcode}} ) { - my $items = $v->{items}; - my $kohafield = $v->{kohafield}; - my ( $table, $column ) = split '\.', $kohafield; - my $output; - while ( my $i = $items->next ) { - my $value = $table eq 'items' - ? $i->$column - : $table eq 'biblio' - ? $i->biblio->$column - : $i->biblioitem->$column; - $output .= " {" . $i->itemnumber . " and " . $i->biblioitem->biblionumber. " => " . $value . "}\n"; + if (%$invalid_av_per_framework) { + new_section('Wrong values linked to authorised values'); + for my $frameworkcode ( keys %$invalid_av_per_framework ) { + while ( my ( $av_category, $v ) = each %{ $invalid_av_per_framework->{$frameworkcode} } ) { + my $items = $v->{items}; + my $kohafield = $v->{kohafield}; + my ( $table, $column ) = split '\.', $kohafield; + my $output; + while ( my $i = $items->next ) { + my $value = + $table eq 'items' ? $i->$column + : $table eq 'biblio' ? $i->biblio->$column + : $i->biblioitem->$column; + $output .= " {" . $i->itemnumber . " and " . $i->biblioitem->biblionumber . " => " . $value . "}\n"; + } + new_item( + sprintf( + "The Framework *%s* is using the authorised value's category *%s*, " + . "but the following %s do not have a value defined ({itemnumber and biblionumber => value }):\n%s", + $frameworkcode, $av_category, $kohafield, $output + ) + ); } - new_item( - sprintf( - "The Framework *%s* is using the authorised value's category *%s*, " - . "but the following %s do not have a value defined ({itemnumber and biblionumber => value }):\n%s", - $frameworkcode, $av_category, $kohafield, $output - ) - ); } } } -} sub CheckItemsTitle { - my $biblios = Koha::Biblios->search({ - -or => [ - title => '', - title => undef, - ] - }); + my $biblios = Koha::Biblios->search( + { + -or => [ + title => '', + title => undef, + ] + } + ); if ( $biblios->count ) { - my ( $title_tag, $title_subtag ) = C4::Biblio::GetMarcFromKohaField( 'biblio.title' ); + my ( $title_tag, $title_subtag ) = C4::Biblio::GetMarcFromKohaField('biblio.title'); my $title_field = $title_tag // ''; - $title_field .= '$'.$title_subtag if $title_subtag; + $title_field .= '$' . $title_subtag if $title_subtag; new_section("Biblio without title $title_field"); while ( my $biblio = $biblios->next ) { - new_item(sprintf "Biblio with biblionumber=%s does not have title defined", $biblio->biblionumber); + new_item( sprintf "Biblio with biblionumber=%s does not have title defined", $biblio->biblionumber ); } new_hint("Edit these bibliographic records to define a title"); } @@ -400,7 +429,7 @@ sub CheckAgeForCategory { -not => { -or => { 'me.dateofbirth' => undef, - -and => { + -and => { 'categorycode.dateofbirthrequired' => undef, 'categorycode.upperagelimit' => undef, } @@ -420,8 +449,8 @@ sub CheckAgeForCategory { my $category = $invalid_patron->category; new_item( sprintf "Patron borrowernumber=%s has an invalid age of %s for their category '%s' (%s to %s)", - $invalid_patron->borrowernumber, $invalid_patron->get_age, $category->categorycode, - $category->dateofbirthrequired // '0', $category->upperagelimit // 'unlimited' + $invalid_patron->borrowernumber, $invalid_patron->get_age, $category->categorycode, + $category->dateofbirthrequired // '0', $category->upperagelimit // 'unlimited' ); } new_hint("You may change the patron's category automatically with misc/cronjobs/update_patrons_category.pl"); @@ -661,4 +690,4 @@ Catch data inconsistencies in Koha database: Note: If no options are provided, all tests will be run. -=cut \ No newline at end of file +=cut -- 2.34.1