View | Details | Raw Unified | Return to bug 36027
Collapse All | Expand All

(-)a/misc/maintenance/search_for_data_inconsistencies.pl (-35 / +283 lines)
Lines 36-47 our %options = ( Link Here
36
    'check-framework' => 0,
36
    'check-framework' => 0,
37
    'check-title'     => 0,
37
    'check-title'     => 0,
38
    'check-age'       => 0,
38
    'check-age'       => 0,
39
    'skip-branch'     => 1,
39
    'check-loop'      => 0,
40
    'skip-auth'       => 1,
40
    'skip-branch'     => 0,
41
    'skip-status'     => 1,
41
    'skip-auth'       => 0,
42
    'skip-framework'  => 1,
42
    'skip-status'     => 0,
43
    'skip-title'      => 1,
43
    'skip-framework'  => 0,
44
    'skip-age'        => 1,
44
    'skip-title'      => 0,
45
    'skip-age'        => 0,
46
    'skip-loop'       => 0,
45
    'help'            => 0,
47
    'help'            => 0,
46
);
48
);
47
49
Lines 53-64 GetOptions( Link Here
53
    'check-framework' => sub { $options{'check-framework'} = 1 },
55
    'check-framework' => sub { $options{'check-framework'} = 1 },
54
    'check-title'     => sub { $options{'check-title'}     = 1 },
56
    'check-title'     => sub { $options{'check-title'}     = 1 },
55
    'check-age'       => sub { $options{'check-age'}       = 1 },
57
    'check-age'       => sub { $options{'check-age'}       = 1 },
56
    'skip-branch'     => sub { $options{'skip-branch'}     = 0 },
58
    'check-loop'      => sub { $options{'check-loop'}      = 1 },
57
    'skip-auth'       => sub { $options{'skip-auth'}       = 0 },
59
    'skip-branch'     => sub { $options{'skip-branch'}     = 1 },
58
    'skip-status'     => sub { $options{'skip-status'}     = 0 },
60
    'skip-auth'       => sub { $options{'skip-auth'}       = 1 },
59
    'skip-framework'  => sub { $options{'skip-framework'}  = 0 },
61
    'skip-status'     => sub { $options{'skip-status'}     = 1 },
60
    'skip-title'      => sub { $options{'skip-title'}      = 0 },
62
    'skip-framework'  => sub { $options{'skip-framework'}  = 1 },
61
    'skip-age'        => sub { $options{'skip-age'}        = 0 },
63
    'skip-title'      => sub { $options{'skip-title'}      = 1 },
64
    'skip-age'        => sub { $options{'skip-age'}        = 1 },
65
    'skip-loop'       => sub { $options{'skip-loop'}       = 1 },
62
    'help'            => sub { $options{'help'}            = 1; },
66
    'help'            => sub { $options{'help'}            = 1; },
63
) or pod2usage(2);    # Print usage if invalid options are provided
67
) or pod2usage(2);    # Print usage if invalid options are provided
64
68
Lines 74-87 GetOptions( Link Here
74
# Print usage and exit if --help flag is provided
78
# Print usage and exit if --help flag is provided
75
pod2usage(1) if $options{'help'};
79
pod2usage(1) if $options{'help'};
76
80
77
# Run tests based on options
81
# Set skip options based on check options
78
set_skip_options();
82
set_skip_options();
79
CheckItemsBranch()     if $options{'check-branch'}    || $options{'skip-branch'};
83
# Set all check options to 1 if none are provided, unless specified skip options
80
CheckItemsAuthHeader() if $options{'check-auth'}      || $options{'skip-auth'};
84
set_all_check_options_if_none_provided();
81
CheckItemsStatus()     if $options{'check-status'}    || $options{'skip-status'};
85
# Run checks based on options
82
CheckItemsFramework()  if $options{'check-framework'} || $options{'skip-framework'};
86
CheckItemsBranch()       if $options{'check-branch'}    && !$options{'skip-branch'};
83
CheckItemsTitle()      if $options{'check-title'}     || $options{'skip-title'};
87
CheckItemsAuthHeader()   if $options{'check-auth'}      && !$options{'skip-auth'};
84
CheckAgeForCategory()  if $options{'check-age'}       || $options{'skip-age'};
88
CheckItemsStatus()       if $options{'check-status'}    && !$options{'skip-status'};
89
CheckItemsFramework()    if $options{'check-framework'} && !$options{'skip-framework'};
90
CheckItemsTitle()        if $options{'check-title'}     && !$options{'skip-title'};
91
CheckAgeForCategory()    if $options{'check-age'}       && !$options{'skip-age'};
92
CheckRelationshipsLoop() if $options{'check-loop'}      && !$options{'skip-loop'};
85
93
86
# Handle invalid options
94
# Handle invalid options
87
sub handle_invalid_options {
95
sub handle_invalid_options {
Lines 92-106 sub handle_invalid_options { Link Here
92
100
93
# Set skip options based on check options
101
# Set skip options based on check options
94
sub set_skip_options {
102
sub set_skip_options {
95
    my $has_check_option = grep { $options{$_} == 1 } grep { /^check-/ } keys %options;
103
    # If at least one check option is provided, set all skip options to 0
104
    my $has_check_option = grep { $options{$_} } grep { /^check-/ } keys %options;
96
    if ($has_check_option) {
105
    if ($has_check_option) {
106
        # if a least one skip option is provided, print a warning
107
        my $has_skip_option = grep { $options{$_} == 1 } grep { /^skip-/ } keys %options;
108
        if ($has_skip_option) {
109
            print("Warning : skip options are ignored when check options are provided\n")
110
        }
111
        # Set all skip options to 0
97
        foreach my $key (keys %options) {
112
        foreach my $key (keys %options) {
98
            $options{$key} = 0 if $key =~ /^skip-/;
113
            if ($key =~ /^skip-/) {
114
                $options{$key} = 0;
115
            }
99
        }
116
        }
100
    }
117
    }
101
    return %options;
118
    return %options;
102
}
119
}
103
120
121
# Set all check options to 1 if none are provided, considering skip options
122
sub set_all_check_options_if_none_provided {
123
    my $any_check_option_provided = grep { $options{$_} } grep { /^check-/ } keys %options;
124
    if (!$any_check_option_provided) {
125
        foreach my $key (keys %options) {
126
            if ($key =~ /^check-/) {
127
                my $skip_key = $key;
128
                $skip_key =~ s/^check-/skip-/;
129
                # Set check option to 1 unless the corresponding skip option indicated
130
                $options{$key} = 1 unless $options{$skip_key};
131
            }
132
        }
133
    }
134
}
135
104
sub CheckItemsBranch {
136
sub CheckItemsBranch {
105
    my $items = Koha::Items->search({ -or => { homebranch => undef, holdingbranch => undef }});
137
    my $items = Koha::Items->search({ -or => { homebranch => undef, holdingbranch => undef }});
106
    if ( $items->count ) { new_section("Not defined items.homebranch and/or items.holdingbranch")}
138
    if ( $items->count ) { new_section("Not defined items.homebranch and/or items.holdingbranch")}
Lines 431-436 sub CheckAgeForCategory { Link Here
431
}
463
}
432
464
433
{
465
{
466
    use Koha::Database;
467
    my $schema = Koha::Database->new->schema;
468
469
    # Loop over all the DBIx::Class classes
470
    for my $class ( sort values %{ $schema->{class_mappings} } ) {
471
472
        # Retrieve the resultset so we can access the columns info
473
        my $rs      = $schema->resultset($class);
474
        my $columns = $rs->result_source->columns_info;
475
476
        # Loop over the columns
477
        while ( my ( $column, $info ) = each %$columns ) {
478
479
            # Next if data type is not date/datetime/timestamp
480
            my $data_type = $info->{data_type};
481
            next unless grep { $data_type =~ m{^$_$} } qw( timestamp datetime date );
482
483
            # Count the invalid dates
484
            my $invalid_dates = $rs->search( { $column => '0000-00-00' } )->count;
485
486
            next unless $invalid_dates;
487
488
            new_section(
489
                "Column " . $rs->result_source->name . "." . $column . " contains $invalid_dates invalid dates" );
490
491
            if ( $invalid_dates > 0 ) {
492
                new_hint("You may change the dates with script: misc/cronjobs/fix_invalid_dates.pl (-c -v)");
493
            }
494
495
        }
496
    }
497
}
498
499
{
500
    my @loop_borrowers_relationships;
501
    my @guarantor_ids = Koha::Patron::Relationships->_resultset->get_column('guarantor_id')->all();
502
    my @guarantee_ids = Koha::Patron::Relationships->_resultset->get_column('guarantee_id')->all();
503
504
    foreach my $guarantor_id (@guarantor_ids) {
505
        foreach my $guarantee_id (@guarantee_ids) {
506
            if ( $guarantor_id == $guarantee_id ) {
507
508
                my $relation_guarantor_id;
509
                my $relation_guarantee_id;
510
                my $size_list;
511
                my $tmp_garantor_id = $guarantor_id;
512
                my @guarantor_ids;
513
514
                do {
515
                    my @relationship_for_go = Koha::Patron::Relationships->search(
516
                        {
517
                            -or => [
518
                                'guarantor_id' => { '=', $tmp_garantor_id },
519
                            ]
520
                        },
521
                    )->as_list;
522
                    $size_list = scalar @relationship_for_go;
523
524
                    foreach my $relation (@relationship_for_go) {
525
                        $relation_guarantor_id = $relation->guarantor_id;
526
                        unless ( grep { $_ == $relation_guarantor_id } @guarantor_ids ) {
527
                            push @guarantor_ids, $relation_guarantor_id;
528
                        }
529
                        $relation_guarantee_id = $relation->guarantee_id;
530
531
                        my @relationship_for_go = Koha::Patron::Relationships->search(
532
                            {
533
                                -or => [
534
                                    'guarantor_id' => { '=', $relation_guarantee_id },
535
                                ]
536
                            },
537
                        )->as_list;
538
                        $size_list = scalar @relationship_for_go;
539
540
                        if ( $guarantor_id == $relation_guarantee_id ) {
541
                            last;
542
                        }
543
544
                        foreach my $relation (@relationship_for_go) {
545
                            $relation_guarantor_id = $relation->guarantor_id;
546
                            unless ( grep { $_ == $relation_guarantor_id } @guarantor_ids ) {
547
                                push @guarantor_ids, $relation_guarantor_id;
548
                            }
549
                            $relation_guarantee_id = $relation->guarantee_id;
550
551
                            if ( $guarantor_id == $relation_guarantee_id ) {
552
                                last;
553
                            }
554
                        }
555
                        if ( $guarantor_id == $relation_guarantee_id ) {
556
                            last;
557
                        }
558
                    }
559
560
                    $tmp_garantor_id = $relation_guarantee_id;
561
                } while ( $guarantor_id != $relation_guarantee_id && $size_list > 0 );
562
563
                if ( $guarantor_id == $relation_guarantee_id ) {
564
                    unless ( grep { join( "", sort @$_ ) eq join( "", sort @guarantor_ids ) }
565
                        @loop_borrowers_relationships )
566
                    {
567
                        push @loop_borrowers_relationships, \@guarantor_ids;
568
                    }
569
                }
570
            }
571
        }
572
    }
573
574
    if ( scalar @loop_borrowers_relationships > 0 ) {
575
        new_section("The list of guarantors who are also guarantee.");
576
        my $count = 0;
577
        foreach my $table (@loop_borrowers_relationships) {
578
            $count++;
579
            print "Loop $count, borrowers id  : ";
580
            foreach my $borrower_id (@$table) {
581
                print "$borrower_id , ";
582
            }
583
            print "\n";
584
        }
585
        new_hint("Relationships that form guarantor loops must be deleted");
586
    }
587
}
588
589
sub CheckRelationshipsLoop {
590
    my @loop_borrowers_relationships;
591
    my @guarantor_ids = Koha::Patron::Relationships->_resultset->get_column('guarantor_id')->all();
592
    my @guarantee_ids = Koha::Patron::Relationships->_resultset->get_column('guarantee_id')->all();
593
594
    foreach my $guarantor_id (@guarantor_ids) {
595
        foreach my $guarantee_id (@guarantee_ids) {
596
            if ( $guarantor_id == $guarantee_id ) {
597
598
                my $relation_guarantor_id;
599
                my $relation_guarantee_id;
600
                my $size_list;
601
                my $tmp_garantor_id = $guarantor_id;
602
                my @guarantor_ids;
603
604
                do {
605
                    my @relationship_for_go = Koha::Patron::Relationships->search(
606
                        {
607
                            -or => [
608
                                'guarantor_id' => { '=', $tmp_garantor_id },
609
                            ]
610
                        },
611
                    )->as_list;
612
                    $size_list = scalar @relationship_for_go;
613
614
                    foreach my $relation (@relationship_for_go) {
615
                        $relation_guarantor_id = $relation->guarantor_id;
616
                        unless ( grep { $_ == $relation_guarantor_id } @guarantor_ids ) {
617
                            push @guarantor_ids, $relation_guarantor_id;
618
                        }
619
                        $relation_guarantee_id = $relation->guarantee_id;
620
621
                        my @relationship_for_go = Koha::Patron::Relationships->search(
622
                            {
623
                                -or => [
624
                                    'guarantor_id' => { '=', $relation_guarantee_id },
625
                                ]
626
                            },
627
                        )->as_list;
628
                        $size_list = scalar @relationship_for_go;
629
630
                        if ( $guarantor_id == $relation_guarantee_id ) {
631
                            last;
632
                        }
633
634
                        foreach my $relation (@relationship_for_go) {
635
                            $relation_guarantor_id = $relation->guarantor_id;
636
                            unless ( grep { $_ == $relation_guarantor_id } @guarantor_ids ) {
637
                                push @guarantor_ids, $relation_guarantor_id;
638
                            }
639
                            $relation_guarantee_id = $relation->guarantee_id;
640
641
                            if ( $guarantor_id == $relation_guarantee_id ) {
642
                                last;
643
                            }
644
                        }
645
                        if ( $guarantor_id == $relation_guarantee_id ) {
646
                            last;
647
                        }
648
                    }
649
650
                    $tmp_garantor_id = $relation_guarantee_id;
651
                } while ( $guarantor_id != $relation_guarantee_id && $size_list > 0 );
652
653
                if ( $guarantor_id == $relation_guarantee_id ) {
654
                    unless ( grep { join( "", sort @$_ ) eq join( "", sort @guarantor_ids ) }
655
                        @loop_borrowers_relationships )
656
                    {
657
                        push @loop_borrowers_relationships, \@guarantor_ids;
658
                    }
659
                }
660
            }
661
        }
662
    }
663
664
    if ( scalar @loop_borrowers_relationships > 0 ) {
665
        new_section("The list of guarantors who are also guarantee.");
666
        my $count = 0;
667
        foreach my $table (@loop_borrowers_relationships) {
668
            $count++;
669
            print "Loop $count, borrowers id  : ";
670
            foreach my $borrower_id (@$table) {
671
                print "$borrower_id , ";
672
            }
673
            print "\n";
674
        }
675
        new_hint("Relationships that form guarantor loops must be deleted");
676
    }
677
}
678
679
sub CheckRelationshipsLoop {
434
    my @loop_borrowers_relationships;
680
    my @loop_borrowers_relationships;
435
    my @guarantor_ids = Koha::Patron::Relationships->_resultset->get_column('guarantor_id')->all();
681
    my @guarantor_ids = Koha::Patron::Relationships->_resultset->get_column('guarantor_id')->all();
436
    my @guarantee_ids = Koha::Patron::Relationships->_resultset->get_column('guarantee_id')->all();
682
    my @guarantee_ids = Koha::Patron::Relationships->_resultset->get_column('guarantee_id')->all();
Lines 551-573 Catch data inconsistencies in Koha database: Link Here
551
    * Bibliographic records without a title
797
    * Bibliographic records without a title
552
    * Invalid MARCXML in bibliographic records
798
    * Invalid MARCXML in bibliographic records
553
    * Patrons with invalid category types due to lower and upper age limits
799
    * Patrons with invalid category types due to lower and upper age limits
800
    * Relationships that form guarantor loops
554
* Any date fields in the database (timestamp, datetime, date) set to 0000-00-00
801
* Any date fields in the database (timestamp, datetime, date) set to 0000-00-00
555
802
556
=head2 OPTIONS
803
=head2 OPTIONS
557
804
558
  --check-branch      Check for items without home or holding library
805
  --check-branch     Check for items without home or holding library
559
  --check-auth        Check for authority records with invalid authority type
806
  --check-auth       Check for authority records with invalid authority type
560
  --check-status      Check for bibliographic records and items without an item type or with an invalid item type
807
  --check-status     Check for bibliographic records and items without an item type or with an invalid item type
561
  --check-framework   Check for invalid values in fields where the framework limits to an authorized value category
808
  --check-framework  Check for invalid values in fields where the framework limits to an authorized value category
562
  --check-title       Check for bibliographic records without a title
809
  --check-title      Check for bibliographic records without a title
563
  --check-age         Check for patrons with invalid age for category
810
  --check-age        Check for patrons with invalid age for category
564
  --skip-branch       Skip checking for items without home or holding library
811
  --check-loop       Check for relationships that form guarantor loops
565
  --skip-auth         Skip checking for authority records with invalid authority type
812
  --skip-branch      Skip checking for items without home or holding library
566
  --skip-status       Skip checking for bibliographic records and items without an item type or with an invalid item type
813
  --skip-auth        Skip checking for authority records with invalid authority type
567
  --skip-framework    Skip checking for invalid values in fields where the framework limits to an authorized value category
814
  --skip-status      Skip checking for bibliographic records and items without an item type or with an invalid item type
568
  --skip-title        Skip checking for bibliographic records without a title
815
  --skip-framework   Skip checking for invalid values in fields where the framework limits to an authorized value category
569
  --skip-age          Skip checking for patrons with invalid age for category
816
  --skip-title       Skip checking for bibliographic records without a title
570
  --help              Print usage information
817
  --skip-age         Skip checking for patrons with invalid age for category
818
  --skip-loop        Skip checking for relationships that form guarantor loops
819
  --help             Print usage information
571
820
572
Note: If no options are provided, all tests will be run.
821
Note: If no options are provided, all tests will be run.
573
822
574
- 

Return to bug 36027