Attachment #44067 for bug #12598

View | Details | Raw Unified | Return to bug 12598
Collapse All | Expand All




package Koha::Patrons::Import;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;
use Text::CSV;

use C4::Members;
use C4::Branch;

sub import_patrons {
    my ($params) = @_;

    my $handle               = $params->{file};
    my $matchpoint           = $params->{matchpoint};
    my $defaults             = $params->{defaults};
    my $ext_preserve         = $params->{preserve_extended_attributes};
    my $overwrite_cardnumber = $params->{overwrite_cardnumber};

    carp("No file handle passed in!") && return unless $handle;

    my $extended            = C4::Context->preference('ExtendedPatronAttributes');
    my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');

    my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
    push( @columnkeys, 'patron_attributes' ) if $extended;

    our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode

    my @feedback;
    my @errors;

    my $imported    = 0;
    my $alreadyindb = 0;
    my $overwritten = 0;
    my $invalid     = 0;
    my $matchpoint_attr_type;

    # use header line to construct key to column map
    my $borrowerline = <$handle>;
    my $status       = $csv->parse($borrowerline);
    ($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline };
    my @csvcolumns = $csv->fields();
    my %csvkeycol;
    my $col = 0;
    foreach my $keycol (@csvcolumns) {

        # columnkeys don't contain whitespace, but some stupid tools add it
        $keycol =~ s/ +//g;
        $csvkeycol{$keycol} = $col++;
    }

    #warn($borrowerline);
    if ($extended) {
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
    }

    push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
    my $today_iso = C4::Dates->new()->output('iso');
    my @criticals = qw(surname branchcode categorycode);    # there probably should be others
    my @bad_dates;                                          # I've had a few.
    my $date_re = C4::Dates->new->regexp('syspref');
    my $iso_re  = C4::Dates->new->regexp('iso');
  LINE: while ( my $borrowerline = <$handle> ) {
        my %borrower;
        my @missing_criticals;
        my $patron_attributes;
        my $status  = $csv->parse($borrowerline);
        my @columns = $csv->fields();
        if ( !$status ) {
            push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
        }
        elsif ( @columns == @columnkeys ) {
            @borrower{@columnkeys} = @columns;

            # MJR: try to fill blanks gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( $borrower{$key} !~ /\S/ ) {
                    $borrower{$key} = $defaults->{$key};
                }
            }
        }
        else {
            # MJR: try to recover gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
                    $borrower{$key} = $columns[ $csvkeycol{$key} ];
                }
                elsif ( $defaults->{$key} ) {
                    $borrower{$key} = $defaults->{$key};
                }
                elsif ( scalar grep { $key eq $_ } @criticals ) {

                    # a critical field is undefined
                    push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
                }
                else {
                    $borrower{$key} = '';
                }
            }
        }

        #warn join(':',%borrower);
        if ( $borrower{categorycode} ) {
            push @missing_criticals,
              {
                key          => 'categorycode',
                line         => $.,
                lineraw      => $borrowerline,
                value        => $borrower{categorycode},
                category_map => 1
              }
              unless GetBorrowercategory( $borrower{categorycode} );
        }
        else {
            push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
        }
        if ( $borrower{branchcode} ) {
            push @missing_criticals,
              {
                key        => 'branchcode',
                line       => $.,
                lineraw    => $borrowerline,
                value      => $borrower{branchcode},
                branch_map => 1
              }
              unless GetBranchName( $borrower{branchcode} );
        }
        else {
            push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
        }
        if (@missing_criticals) {
            foreach (@missing_criticals) {
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
                $_->{surname}        = $borrower{surname}        || 'UNDEF';
            }
            $invalid++;
            ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };

            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
            next LINE;
        }
        if ($extended) {
            my $attr_str = $borrower{patron_attributes};
            $attr_str =~ s/\xe2\x80\x9c/"/g;    # fixup double quotes in case we are passed smart quotes
            $attr_str =~ s/\xe2\x80\x9d/"/g;
            push @feedback, { feedback => 1, name => 'attribute string', value => $attr_str };
            delete $borrower{patron_attributes}
              ;    # not really a field in borrowers, so we don't want to pass it to ModMember.
            $patron_attributes = extended_attributes_code_value_arrayref($attr_str);
        }

        # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
        foreach (qw(dateofbirth dateenrolled dateexpiry)) {
            my $tempdate = $borrower{$_} or next;
            if ( $tempdate =~ /$date_re/ ) {
                $borrower{$_} = format_date_in_iso($tempdate);
            }
            elsif ( $tempdate =~ /$iso_re/ ) {
                $borrower{$_} = $tempdate;
            }
            else {
                $borrower{$_} = '';
                push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
            }
        }
        $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
        $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
          unless $borrower{dateexpiry};
        my $borrowernumber;
        my $member;
        if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
            $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
            if ($member) {
                $borrowernumber = $member->{'borrowernumber'};
            }
        }
        elsif ($extended) {
            if ( defined($matchpoint_attr_type) ) {
                foreach my $attr (@$patron_attributes) {
                    if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
                        last;
                    }
                }
            }
        }

        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
            push @errors,
              {
                invalid_cardnumber => 1,
                borrowernumber     => $borrowernumber,
                cardnumber         => $borrower{cardnumber}
              };
            $invalid++;
            next;
        }

        if ($borrowernumber) {

            # borrower exists
            unless ($overwrite_cardnumber) {
                $alreadyindb++;
                push(
                    @feedback,
                    {
                        already_in_db => 1,
                        value         => $borrower{'surname'} . ' / ' . $borrowernumber
                    }
                );
                next LINE;
            }
            $borrower{'borrowernumber'} = $borrowernumber;
            for my $col ( keys %borrower ) {

                # use values from extant patron unless our csv file includes this column or we provided a default.
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.

                # The password is always encrypted, skip it!
                next if $col eq 'password';

                unless ( exists( $csvkeycol{$col} ) || $defaults->{$col} ) {
                    $borrower{$col} = $member->{$col} if ( $member->{$col} );
                }
            }
            unless ( ModMember(%borrower) ) {
                $invalid++;

                push(
                    @errors,
                    {
                        name  => 'lastinvalid',
                        value => $borrower{'surname'} . ' / ' . $borrowernumber
                    }
                );
                next LINE;
            }
            if ( $borrower{debarred} ) {

                # Check to see if this debarment already exists
                my $debarrments = GetDebarments(
                    {
                        borrowernumber => $borrowernumber,
                        expiration     => $borrower{debarred},
                        comment        => $borrower{debarredcomment}
                    }
                );

                # If it doesn't, then add it!
                unless (@$debarrments) {
                    AddDebarment(
                        {
                            borrowernumber => $borrowernumber,
                            expiration     => $borrower{debarred},
                            comment        => $borrower{debarredcomment}
                        }
                    );
                }
            }
            if ($extended) {
                if ($ext_preserve) {
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
                    $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes );
                }
                push @errors, { unknown_error => 1 }
                  unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
            }
            $overwritten++;
            push(
                @feedback,
                {
                    feedback => 1,
                    name     => 'lastoverwritten',
                    value    => $borrower{'surname'} . ' / ' . $borrowernumber
                }
            );
        }
        else {
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
            # At least this is closer to AddMember than in members/memberentry.pl
            if ( !$borrower{'cardnumber'} ) {
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
            }
            if ( $borrowernumber = AddMember(%borrower) ) {

                if ( $borrower{debarred} ) {
                    AddDebarment(
                        {
                            borrowernumber => $borrowernumber,
                            expiration     => $borrower{debarred},
                            comment        => $borrower{debarredcomment}
                        }
                    );
                }

                if ($extended) {
                    SetBorrowerAttributes( $borrowernumber, $patron_attributes );
                }

                if ($set_messaging_prefs) {
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
                        {
                            borrowernumber => $borrowernumber,
                            categorycode   => $borrower{categorycode}
                        }
                    );
                }

                $imported++;
                push(
                    @feedback,
                    {
                        feedback => 1,
                        name     => 'lastimported',
                        value    => $borrower{'surname'} . ' / ' . $borrowernumber
                    }
                );
            }
            else {
                $invalid++;
                push @errors, { unknown_error => 1 };
                push(
                    @errors,
                    {
                        name  => 'lastinvalid',
                        value => $borrower{'surname'} . ' / AddMember',
                    }
                );
            }
        }
    }

    return {
        feedback      => \@feedback,
        errors        => \@errors,
        imported      => $imported,
        overwritten   => $overwritten,
        already_in_db => $alreadyindb,
        invalid       => $invalid,
    };
}

END { }    # module clean-up code here (global destructor)

1;

__END__

=head1 AUTHOR

Koha Team

=cut




     <div class="yui-u first">
<h1>Import patrons</h1>
[% IF ( uploadborrowers ) %]
    <h5>Import results :</h5>
<ul>
	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
	<li>[% total %] records parsed</li>
	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
</ul>
  [% IF ( FEEDBACK ) %]
  <br /><br />
    <div>
    <h5>Feedback:</h5>
    <ul class="feedback">
    [% FOREACH FEEDBAC IN FEEDBACK %]
    <li>
        [% IF ( FEEDBAC.filename ) %]Parsing upload file <span class="filename">[% FEEDBAC.filename %]</span>
        [% ELSIF ( FEEDBAC.backend ) %]Upload parsed using [% FEEDBAC.backend %]
        [% ELSIF ( FEEDBAC.headerrow ) %]These fields found: [% FEEDBAC.value %]
        [% ELSE %][% FEEDBAC.name %] : [% FEEDBAC.value %]
        [% END %]
    </li>
    [% END %]
    </ul>
    </div>
  [% END %]
  [% IF ( ERRORS ) %]
  <br /><br />
    <div>
    <h5>Error analysis:</h5>
    <ul>
        <li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
        <li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
        <li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
        <li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
        <li>[% total %] records parsed</li>
        <li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
    </ul>

    [% IF ( feedback ) %]
        <br /><br />

        <div>
            <h5>Feedback:</h5>
                <ul class="feedback">
                    [% FOREACH f IN feedback %]
                        <li>
                            [% IF ( f.filename ) %]
                                Parsing upload file <span class="filename">[% f.filename %]</span>
                            [% ELSIF ( f.backend ) %]
                                Upload parsed using [% f.backend %]
                            [% ELSIF ( f.headerrow ) %]
                                These fields found: [% f.value %]
                            [% ELSIF ( f.already_in_db ) %]
                                Patron already in database: [% f.value %]
                            [% ELSE %]
                                [% f.name %] : [% f.value %]
                            [% END %]
                        </li>
                    [% END %]
                </ul>
        </div>
    [% END %]

    [% IF ( errors ) %]
        <br /><br />

        <div>
            <h5>Error analysis:</h5>
            <ul>
                [% FOREACH e IN errors %]
                    [% IF ( e.badheader ) %]<li>Header row could not be parsed</li>[% END %]

                    [% FOREACH missing_critical IN e.missing_criticals %]
                        <li class="line_error">
                            Line <span class="linenumber">[% missing_critical.line %]</span>

                            [% IF ( missing_critical.badparse ) %]
                                could not be parsed!
                            [% ELSIF ( missing_critical.bad_date ) %]
                                has &quot;[% missing_critical.key %]&quot; in unrecognized format: &quot;[% missing_critical.value %]&quot;
                            [% ELSE %]
                                Critical field &quot;[% missing_critical.key %]&quot;

                                [% IF ( missing_critical.branch_map ) %]
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
                                [% ELSIF ( missing_critical.category_map ) %]
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
                                [% ELSE %]
                                    missing
                                [% END %]

                                (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
                            [% END %]

                            <br/>
                            <code>[% missing_critical.lineraw %]</code>
                        </li>
                    [% END %]

                    [% IF e.invalid_cardnumber %]
                        <li class="line_error">
                            Cardnumber [% e.cardnumber %] is not a valid cardnumber
                            [% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %]
                        </li>
                    [% END %]
                [% END %]
            </ul>
        </div>
            <br /><code>[% missing_critical.lineraw %]</code>
        </li>
        [% END %]
        [% IF ERROR.invalid_cardnumber %]
            <li class="line_error">
                Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber
                [% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %]
            </li>
        [% END %]
    [% END %]
    </ul>
    </div>
  [% END %]
[% ELSE %]
    <ul>
        <li>Select a file to import into the borrowers table</li>
        <li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
    </ul>

    <form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
        <fieldset class="rows">
            <legend>Import into the borrowers table</legend>

            <ol>
                <li>
                    <label for="uploadborrowers">Select the file to import: </label>
                    <input type="file" id="uploadborrowers" name="uploadborrowers" />
                </li>
            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>Field to use for record matching</legend>
            <ol>
                <li class="radio">
                    <select name="matchpoint" id="matchpoint">
                        <option value="cardnumber">Cardnumber</option>
                        [% FOREACH matchpoint IN matchpoints %]
                            <option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
                        [% END %]
                    </select>
                </li>
            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>Default values</legend>

            <ol>
                [% FOREACH borrower_field IN borrower_fields %]

                    [% SWITCH borrower_field.field %]
                    [% CASE 'branchcode' %]
                        <li>
                            <label class="description" for="branchcode">[% borrower_field.description %]: </label>
                            <select id="branchcode" name="branchcode">
                                <option value="" selected="selected"></option>
                                [% FOREACH branch IN branches %]
                                    <option value="[% branch.branchcode %]">[% branch.branchname %]</option>
                                [% END %]
                            </select>
                            <span class="field_hint">[% borrower_field.field %]</span>
                        </li>
                    [% CASE 'categorycode' %]
                        <li>
                            <label class="description" for="categorycode">[% borrower_field.description %]: </label>
                            <select id="categorycode" name="categorycode">
                                <option value="" selected="selected"></option>
                                [% FOREACH category IN categories %]
                                    <option value="[% category.categorycode %]">[% category.description %]</option>
                                [% END %]
                            </select>
                            <span class="field_hint">[% borrower_field.field %]</span>
                        </li>
                    [% CASE %]
                        <li>
                            <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
                            <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" />
                            <span class="field_hint">[% borrower_field.field %]</span>
                        </li>
                    [% END %]

                [% END %]

                [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
                    <li>
                        <label class="description" for="patron_attributes">Patron attributes: </label>
                        <input id="patron_attributes" name="patron_attributes" />
                        <span class="field_hint">patron_attributes</span>
                    </li>
                [% END %]

            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>If matching record is already in the borrowers table:</legend>

            <ol>
                <li class="radio">
                    <input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
                </li>

                <li class="radio">
                    <input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
                </li>
            </ol>
        </fieldset>

        [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
            <fieldset class="rows">
                <legend>Patron attributes</legend>

                <ol>
                    <li class="radio">
                        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
                    </li>

                    <li class="radio">
                        <input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
                    </li>
                </ol>
            </fieldset>
        [% END %]

        <fieldset class="action"><input type="submit" value="Import" /></fieldset>
    </form>
    <li>
        <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
        <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" /><span class="field_hint">[% borrower_field.field %]</span>
    </li>
  [% END %]
[% END %]
[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
    <li>
        <label class="description" for="patron_attributes">Patron attributes: </label>
        <input id="patron_attributes" name="patron_attributes" />
        <span class="field_hint">patron_attributes</span>
    </li>
[% END %]

</div>

<div class="yui-u">
    <h2>Notes:</h2>
    <ul>
        <li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>

        <li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b>  Values are comma-separated.</li>

        <li>
            OR choose which fields you want to supply from the following list:
            <ul>
                <li>
                    [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
                </li>
            </ul>
        </li>

        [% IF ( ExtendedPatronAttributes ) %]
            <li>
                If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>&quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</b>.  The second syntax would be required if the data might have a comma in it, like a date string.
            </li>
        [% END %]

        <li>
            The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.
        </li>

        <li>
            'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
	<fieldset class="rows">
    <legend>Patron attributes</legend>
    <ol><li class="radio">
        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
        </li>

        <li>
            Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.  Alternatively,
you can supply dates in ISO format (e.g., '2010-10-28').
        </li>
    </ul>

    [% END %]
	<fieldset class="action"><input type="submit" value="Import" /></fieldset>
</form>
[% END %]
</div>
</div>
</div>
</div>

<div class="yui-b noprint">
    [% INCLUDE 'tools-menu.inc' %]
</div>
</div>

<li>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>&quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</b>.  The second syntax would be required if the data might have a comma in it, like a date string.
</li>
[% END %]
<li>The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.</li>
<li>'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).</li>
<li>Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.  Alternatively,
you can supply dates in ISO format (e.g., '2010-10-28').
</li>
</ul>

     </div>
    </div>
   </div>
  </div>
  <div class="yui-b noprint">
[% INCLUDE 'tools-menu.inc' %]
  </div>
 </div>
[% INCLUDE 'intranet-bottom.inc' %]




# dates should be in the format you have set up Koha to expect
# branchcode and categorycode need to be valid

use Modern::Perl;
use warnings;

use C4::Auth;
use C4::Output;
use C4::Context;
use C4::Branch qw/GetBranchesLoop GetBranchName/;
use C4::Members;
use C4::Members::Attributes qw(:all);
use C4::Members::AttributeTypes;
use C4::Members::Messaging;
use C4::Reports::Guided;
use C4::Templates;

use C4::Branch;
use C4::Members;

use Koha::Patrons::Import qw(import_patrons);

use Text::CSV;


# use encoding 'utf8';    # don't do this

my ( @errors, @feedback );
my $extended = C4::Context->preference('ExtendedPatronAttributes');

my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
push( @columnkeys, 'patron_attributes' ) if $extended;
if ($extended) {
    push @columnkeys, 'patron_attributes';
}

my $input = CGI->new();
our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode

#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX


$template->param( borrower_fields => $columns );

if ( $input->param('sample') ) {
    our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode
    print $input->header(
        -type       => 'application/vnd.sun.xml.calc',    # 'application/vnd.ms-excel' ?
        -attachment => 'patron_import.csv',

    print $csv->string, "\n";
    exit 1;
}

my $uploadborrowers = $input->param('uploadborrowers');
my $matchpoint      = $input->param('matchpoint');
if ($matchpoint) {
    $matchpoint =~ s/^patron_attribute_//;
}
my $overwrite_cardnumber = $input->param('overwrite_cardnumber');

$template->param( SCRIPT_NAME => '/cgi-bin/koha/tools/import_borrowers.pl' );

if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
    my $handle   = $input->upload('uploadborrowers');
    my $handle     = $input->upload('uploadborrowers');
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
    foreach ( keys %$uploadinfo ) {
        push @feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
    }
    my $imported    = 0;
    my $alreadyindb = 0;
    my $overwritten = 0;
    my $invalid     = 0;
    my $matchpoint_attr_type;
    my %defaults = $input->Vars;

    my $return = Koha::Patrons::Import::import_patrons(
        {
            file                         => $handle,
            defaults                     => \%defaults,
            matchpoint                   => $matchpoint,
            overwrite_cardnumber         => $input->param('overwrite_cardnumber'),
            preserve_extended_attributes => $input->param('ext_preserve') || 0,
    foreach my $keycol (@csvcolumns) {

        # columnkeys don't contain whitespace, but some stupid tools add it
        $keycol =~ s/ +//g;
        $csvkeycol{$keycol} = $col++;
    }

    #warn($borrowerline);
    my $ext_preserve = $input->param('ext_preserve') || 0;
    if ($extended) {
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
    }

    push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
    my $today_iso = output_pref( { dt => dt_from_string, dateonly => 1, dateformat => 'iso' } );
    my @criticals = qw(surname branchcode categorycode);    # there probably should be others
    my @bad_dates;                                          # I've had a few.
  LINE: while ( my $borrowerline = <$handle> ) {
        my %borrower;
        my @missing_criticals;
        my $patron_attributes;
        my $status  = $csv->parse($borrowerline);
        my @columns = $csv->fields();
        if ( !$status ) {
            push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
        }
        elsif ( @columns == @columnkeys ) {
            @borrower{@columnkeys} = @columns;

            # MJR: try to fill blanks gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( $borrower{$key} !~ /\S/ ) {
                    $borrower{$key} = $defaults{$key};
                }
            }
        }
        else {
            # MJR: try to recover gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
                    $borrower{$key} = $columns[ $csvkeycol{$key} ];
                }
                elsif ( $defaults{$key} ) {
                    $borrower{$key} = $defaults{$key};
                }
                elsif ( scalar grep { $key eq $_ } @criticals ) {

                    # a critical field is undefined
                    push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
                }
                else {
                    $borrower{$key} = '';
                }
            }
        }

        #warn join(':',%borrower);
        if ( $borrower{categorycode} ) {
            push @missing_criticals,
              {
                key          => 'categorycode',
                line         => $.,
                lineraw      => $borrowerline,
                value        => $borrower{categorycode},
                category_map => 1
              }
              unless GetBorrowercategory( $borrower{categorycode} );
        }
        else {
            push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
        }
        if ( $borrower{branchcode} ) {
            push @missing_criticals,
              {
                key        => 'branchcode',
                line       => $.,
                lineraw    => $borrowerline,
                value      => $borrower{branchcode},
                branch_map => 1
              }
              unless GetBranchName( $borrower{branchcode} );
        }
        else {
            push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
        }
        if (@missing_criticals) {
            foreach (@missing_criticals) {
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
                $_->{surname}        = $borrower{surname}        || 'UNDEF';
            }
            $invalid++;
            ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };

            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
            next LINE;
        }
        if ($extended) {
            my $attr_str = $borrower{patron_attributes};
            $attr_str =~ s/\xe2\x80\x9c/"/g;    # fixup double quotes in case we are passed smart quotes
            $attr_str =~ s/\xe2\x80\x9d/"/g;
            push @feedback,
              { feedback => 1, name => 'attribute string', value => $attr_str, filename => $uploadborrowers };
            delete $borrower{patron_attributes}
              ;    # not really a field in borrowers, so we don't want to pass it to ModMember.
            $patron_attributes = extended_attributes_code_value_arrayref($attr_str);
        }

        # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
        foreach (qw(dateofbirth dateenrolled dateexpiry)) {
            my $tempdate = $borrower{$_} or next;
            $tempdate =
              eval { output_pref( { dt => dt_from_string($tempdate), dateonly => 1, dateformat => 'iso' } ); };
            if ($tempdate) {
                $borrower{$_} = $tempdate;
            }
            else {
                $borrower{$_} = '';
                push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
            }
        }
    );
        $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
          unless $borrower{dateexpiry};
        my $borrowernumber;
        my $member;
        if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
            $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
            if ($member) {
                $borrowernumber = $member->{'borrowernumber'};
            }
        }
        elsif ( ( $matchpoint eq 'userid' ) && ( $borrower{'userid'} ) ) {
            $member = GetMember( 'userid' => $borrower{'userid'} );
            if ($member) {
                $borrowernumber = $member->{'borrowernumber'};
            }
        }
        elsif ($extended) {
            if ( defined($matchpoint_attr_type) ) {
                foreach my $attr (@$patron_attributes) {
                    if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
                        last;
                    }
                }
            }
        }

        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
            push @errors,
              {
                invalid_cardnumber => 1,
                borrowernumber     => $borrowernumber,
                cardnumber         => $borrower{cardnumber}
              };
            $invalid++;
            next;
        }

        if ($borrowernumber) {

            # borrower exists
            unless ($overwrite_cardnumber) {
                $alreadyindb++;
                $template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber );
                next LINE;
            }
            $borrower{'borrowernumber'} = $borrowernumber;
            for my $col ( keys %borrower ) {

                # use values from extant patron unless our csv file includes this column or we provided a default.
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.

                # The password is always encrypted, skip it!
                next if $col eq 'password';

                unless ( exists( $csvkeycol{$col} ) || $defaults{$col} ) {
                    $borrower{$col} = $member->{$col} if ( $member->{$col} );
                }
            }
            unless ( ModMember(%borrower) ) {
                $invalid++;

                # until we have better error trapping, we have no way of knowing why ModMember errored out...
                push @errors, { unknown_error => 1 };
                $template->param( 'lastinvalid' => $borrower{'surname'} . ' / ' . $borrowernumber );
                next LINE;
            }
            if ( $borrower{debarred} ) {

                # Check to see if this debarment already exists
                my $debarrments = GetDebarments(
                    {
                        borrowernumber => $borrowernumber,
                        expiration     => $borrower{debarred},
                        comment        => $borrower{debarredcomment}
                    }
                );

                # If it doesn't, then add it!
                unless (@$debarrments) {
                    AddDebarment(
                        {
                            borrowernumber => $borrowernumber,
                            expiration     => $borrower{debarred},
                            comment        => $borrower{debarredcomment}
                        }
                    );
                }
            }
            if ($extended) {
                if ($ext_preserve) {
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
                    $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes )
                      ;    #TODO: expose repeatable options in template
                }
                push @errors, { unknown_error => 1 }
                  unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
            }
            $overwritten++;
            $template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber );
        }
        else {
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
            # At least this is closer to AddMember than in members/memberentry.pl
            if ( !$borrower{'cardnumber'} ) {
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
            }
            if ( $borrowernumber = AddMember(%borrower) ) {

    my $feedback    = $return->{feedback};
    my $errors      = $return->{errors};
    my $imported    = $return->{imported};
    my $overwritten = $return->{overwritten};
    my $alreadyindb = $return->{already_in_db};
    my $invalid     = $return->{invalid};
                        }
                    );
                }

    my $uploadinfo = $input->uploadInfo($uploadborrowers);
    foreach ( keys %$uploadinfo ) {
        push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
    }

    push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
                        {
                            borrowernumber => $borrowernumber,
                            categorycode   => $borrower{categorycode}
                        }
                    );
                }

                $imported++;
                $template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber );
            }
            else {
                $invalid++;
                push @errors, { unknown_error => 1 };
                $template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember' );
            }
        }
    }
    (@errors)   and $template->param( ERRORS   => \@errors );
    (@feedback) and $template->param( FEEDBACK => \@feedback );
    $template->param(
        uploadborrowers => 1,
        errors          => $errors,
        feedback        => $feedback,
        imported        => $imported,
        overwritten     => $overwritten,
        alreadyindb     => $alreadyindb,
        invalid         => $invalid,
        total           => $imported + $alreadyindb + $invalid + $overwritten,
    );

}
- 

Return to bug 12598

Line 0 Link Here

(-)a/Koha/Patrons/Import.pm (+370 lines)
		1	package Koha::Patrons::Import;
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it under the
		6	# terms of the GNU General Public License as published by the Free Software
		7	# Foundation; either version 3 of the License, or (at your option) any later
		8	# version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		13	#
		14	# You should have received a copy of the GNU General Public License along
		15	# with Koha; if not, write to the Free Software Foundation, Inc.,
		16	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		17
		18	use Modern::Perl;
		19
		20	use Carp;
		21	use Text::CSV;
		22
		23	use C4::Members;
		24	use C4::Branch;
		25
		26	sub import_patrons {
		27	my ($params) = @_;
		28
		29	my $handle = $params->{file};
		30	my $matchpoint = $params->{matchpoint};
		31	my $defaults = $params->{defaults};
		32	my $ext_preserve = $params->{preserve_extended_attributes};
		33	my $overwrite_cardnumber = $params->{overwrite_cardnumber};
		34
		35	carp("No file handle passed in!") && return unless $handle;
		36
		37	my $extended = C4::Context->preference('ExtendedPatronAttributes');
		38	my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');
		39
		40	my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
		41	push( @columnkeys, 'patron_attributes' ) if $extended;
		42
		43	our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode
		44
		45	my @feedback;
		46	my @errors;
		47
		48	my $imported = 0;
		49	my $alreadyindb = 0;
		50	my $overwritten = 0;
		51	my $invalid = 0;
		52	my $matchpoint_attr_type;
		53
		54	# use header line to construct key to column map
		55	my $borrowerline = <$handle>;
		56	my $status = $csv->parse($borrowerline);
		57	($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline };
		58	my @csvcolumns = $csv->fields();
		59	my %csvkeycol;
		60	my $col = 0;
		61	foreach my $keycol (@csvcolumns) {
		62
		63	# columnkeys don't contain whitespace, but some stupid tools add it
		64	$keycol =~ s/ +//g;
65	$csvkeycol{$keycol} = $col++;
66	}
67
68	#warn($borrowerline);
69	if ($extended) {
70	$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
71	}
72
73	push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
74	my $today_iso = C4::Dates->new()->output('iso');
75	my @criticals = qw(surname branchcode categorycode); # there probably should be others
76	my @bad_dates; # I've had a few.
77	my $date_re = C4::Dates->new->regexp('syspref');
78	my $iso_re = C4::Dates->new->regexp('iso');
79	LINE: while ( my $borrowerline = <$handle> ) {
80	my %borrower;
81	my @missing_criticals;
82	my $patron_attributes;
83	my $status = $csv->parse($borrowerline);
84	my @columns = $csv->fields();
85	if ( !$status ) {
86	push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
87	}
88	elsif ( @columns == @columnkeys ) {
89	@borrower{@columnkeys} = @columns;
90
91	# MJR: try to fill blanks gracefully by using default values
92	foreach my $key (@columnkeys) {
93	if ( $borrower{$key} !~ /\S/ ) {
94	$borrower{$key} = $defaults->{$key};
95	}
96	}
97	}
98	else {
99	# MJR: try to recover gracefully by using default values
100	foreach my $key (@columnkeys) {
101	if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
102	$borrower{$key} = $columns[ $csvkeycol{$key} ];
103	}
104	elsif ( $defaults->{$key} ) {
105	$borrower{$key} = $defaults->{$key};
106	}
107	elsif ( scalar grep { $key eq $_ } @criticals ) {
108
109	# a critical field is undefined
110	push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
111	}
112	else {
113	$borrower{$key} = '';
114	}
115	}
116	}
117
118	#warn join(':',%borrower);
119	if ( $borrower{categorycode} ) {
120	push @missing_criticals,
121	{
122	key => 'categorycode',
123	line => $.,
124	lineraw => $borrowerline,
125	value => $borrower{categorycode},
126	category_map => 1
127	}
128	unless GetBorrowercategory( $borrower{categorycode} );
129	}
130	else {
131	push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
132	}
133	if ( $borrower{branchcode} ) {
134	push @missing_criticals,
135	{
136	key => 'branchcode',
137	line => $.,
138	lineraw => $borrowerline,
139	value => $borrower{branchcode},
140	branch_map => 1
141	}
142	unless GetBranchName( $borrower{branchcode} );
143	}
144	else {
145	push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
146	}
147	if (@missing_criticals) {
148	foreach (@missing_criticals) {
149	$_->{borrowernumber} = $borrower{borrowernumber} \|\| 'UNDEF';
150	$_->{surname} = $borrower{surname} \|\| 'UNDEF';
151	}
152	$invalid++;
153	( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };
154
155	# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance.
156	next LINE;
157	}
158	if ($extended) {
159	my $attr_str = $borrower{patron_attributes};
160	$attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes
161	$attr_str =~ s/\xe2\x80\x9d/"/g;
162	push @feedback, { feedback => 1, name => 'attribute string', value => $attr_str };
163	delete $borrower{patron_attributes}
164	; # not really a field in borrowers, so we don't want to pass it to ModMember.
165	$patron_attributes = extended_attributes_code_value_arrayref($attr_str);
166	}
167
168	# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
169	foreach (qw(dateofbirth dateenrolled dateexpiry)) {
170	my $tempdate = $borrower{$_} or next;
171	if ( $tempdate =~ /$date_re/ ) {
172	$borrower{$_} = format_date_in_iso($tempdate);
173	}
174	elsif ( $tempdate =~ /$iso_re/ ) {
175	$borrower{$_} = $tempdate;
176	}
177	else {
178	$borrower{$_} = '';
179	push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
180	}
181	}
182	$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
183	$borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
184	unless $borrower{dateexpiry};
185	my $borrowernumber;
186	my $member;
187	if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
188	$member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
189	if ($member) {
190	$borrowernumber = $member->{'borrowernumber'};
191	}
192	}
193	elsif ($extended) {
194	if ( defined($matchpoint_attr_type) ) {
195	foreach my $attr (@$patron_attributes) {
196	if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
197	my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
198	$borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
199	last;
200	}
201	}
202	}
203	}
204
205	if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
206	push @errors,
207	{
208	invalid_cardnumber => 1,
209	borrowernumber => $borrowernumber,
210	cardnumber => $borrower{cardnumber}
211	};
212	$invalid++;
213	next;
214	}
215
216	if ($borrowernumber) {
217
218	# borrower exists
219	unless ($overwrite_cardnumber) {
220	$alreadyindb++;
221	push(
222	@feedback,
223	{
224	already_in_db => 1,
225	value => $borrower{'surname'} . ' / ' . $borrowernumber
226	}
227	);
228	next LINE;
229	}
230	$borrower{'borrowernumber'} = $borrowernumber;
231	for my $col ( keys %borrower ) {
232
233	# use values from extant patron unless our csv file includes this column or we provided a default.
234	# FIXME : You cannot update a field with a perl-evaluated false value using the defaults.
235
236	# The password is always encrypted, skip it!
237	next if $col eq 'password';
238
239	unless ( exists( $csvkeycol{$col} ) \|\| $defaults->{$col} ) {
240	$borrower{$col} = $member->{$col} if ( $member->{$col} );
241	}
242	}
243	unless ( ModMember(%borrower) ) {
244	$invalid++;
245
246	push(
247	@errors,
248	{
249	name => 'lastinvalid',
250	value => $borrower{'surname'} . ' / ' . $borrowernumber
251	}
252	);
253	next LINE;
254	}
255	if ( $borrower{debarred} ) {
256
257	# Check to see if this debarment already exists
258	my $debarrments = GetDebarments(
259	{
260	borrowernumber => $borrowernumber,
261	expiration => $borrower{debarred},
262	comment => $borrower{debarredcomment}
263	}
264	);
265
266	# If it doesn't, then add it!
267	unless (@$debarrments) {
268	AddDebarment(
269	{
270	borrowernumber => $borrowernumber,
271	expiration => $borrower{debarred},
272	comment => $borrower{debarredcomment}
273	}
274	);
275	}
276	}
277	if ($extended) {
278	if ($ext_preserve) {
279	my $old_attributes = GetBorrowerAttributes($borrowernumber);
280	$patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes );
281	}
282	push @errors, { unknown_error => 1 }
283	unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
284	}
285	$overwritten++;
286	push(
287	@feedback,
288	{
289	feedback => 1,
290	name => 'lastoverwritten',
291	value => $borrower{'surname'} . ' / ' . $borrowernumber
292	}
293	);
294	}
295	else {
296	# FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
297	# At least this is closer to AddMember than in members/memberentry.pl
298	if ( !$borrower{'cardnumber'} ) {
299	$borrower{'cardnumber'} = fixup_cardnumber(undef);
300	}
301	if ( $borrowernumber = AddMember(%borrower) ) {
302
303	if ( $borrower{debarred} ) {
304	AddDebarment(
305	{
306	borrowernumber => $borrowernumber,
307	expiration => $borrower{debarred},
308	comment => $borrower{debarredcomment}
309	}
310	);
311	}
312
313	if ($extended) {
314	SetBorrowerAttributes( $borrowernumber, $patron_attributes );
315	}
316
317	if ($set_messaging_prefs) {
318	C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
319	{
320	borrowernumber => $borrowernumber,
321	categorycode => $borrower{categorycode}
322	}
323	);
324	}
325
326	$imported++;
327	push(
328	@feedback,
329	{
330	feedback => 1,
331	name => 'lastimported',
332	value => $borrower{'surname'} . ' / ' . $borrowernumber
333	}
334	);
335	}
336	else {
337	$invalid++;
338	push @errors, { unknown_error => 1 };
339	push(
340	@errors,
341	{
342	name => 'lastinvalid',
343	value => $borrower{'surname'} . ' / AddMember',
344	}
345	);
346	}
347	}
348	}
349
350	return {
351	feedback => \@feedback,
352	errors => \@errors,
353	imported => $imported,
354	overwritten => $overwritten,
355	already_in_db => $alreadyindb,
356	invalid => $invalid,
357	};
358	}
359
360	END { } # module clean-up code here (global destructor)
361
362	1;
363
364	__END__
365
366	=head1 AUTHOR
367
368	Koha Team
369
370	=cut

Lines 33-220 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt (-172 / +233 lines)
33	<div class="yui-u first">	33	<div class="yui-u first">
34	<h1>Import patrons</h1>	34	<h1>Import patrons</h1>
35	[% IF ( uploadborrowers ) %]	35	[% IF ( uploadborrowers ) %]
36	<h5>Import results :</h5>	36	<h5>Import results :</h5>
37	<ul>
38	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
39	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
40	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
41	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
42	<li>[% total %] records parsed</li>
43	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
44	</ul>
45	[% IF ( FEEDBACK ) %]
46	<br /><br />
47	<div>
48	<h5>Feedback:</h5>
49	<ul class="feedback">
50	[% FOREACH FEEDBAC IN FEEDBACK %]
51	<li>
52	[% IF ( FEEDBAC.filename ) %]Parsing upload file <span class="filename">[% FEEDBAC.filename %]</span>
53	[% ELSIF ( FEEDBAC.backend ) %]Upload parsed using [% FEEDBAC.backend %]
54	[% ELSIF ( FEEDBAC.headerrow ) %]These fields found: [% FEEDBAC.value %]
55	[% ELSE %][% FEEDBAC.name %] : [% FEEDBAC.value %]
56	[% END %]
57	</li>
58	[% END %]
59	</ul>
60	</div>
61	[% END %]
62	[% IF ( ERRORS ) %]
63	<br /><br />
64	<div>
65	<h5>Error analysis:</h5>
66	<ul>	37	<ul>
67	[% FOREACH ERROR IN ERRORS %]	38	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
68	[% IF ( ERROR.badheader ) %]<li>Header row could not be parsed</li>[% END %]	39	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
69	[% FOREACH missing_critical IN ERROR.missing_criticals %]	40	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
70	<li class="line_error">	41	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
71	Line <span class="linenumber">[% missing_critical.line %]</span>	42	<li>[% total %] records parsed</li>
72	[% IF ( missing_critical.badparse ) %]	43	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
73	could not be parsed!	44	</ul>
74	[% ELSIF ( missing_critical.bad_date ) %]	45
75	has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]"	46	[% IF ( feedback ) %]
76	[% ELSE %]	47	<br /><br />
77	Critical field "[% missing_critical.key %]"	48
78	[% IF ( missing_critical.branch_map ) %]has unrecognized value "[% missing_critical.value %]"	49	<div>
79	[% ELSIF ( missing_critical.category_map ) %]has unrecognized value "[% missing_critical.value %]"	50	<h5>Feedback:</h5>
80	[% ELSE %]missing	51	<ul class="feedback">
		52	[% FOREACH f IN feedback %]
		53	<li>
		54	[% IF ( f.filename ) %]
		55	Parsing upload file <span class="filename">[% f.filename %]</span>
		56	[% ELSIF ( f.backend ) %]
		57	Upload parsed using [% f.backend %]
		58	[% ELSIF ( f.headerrow ) %]
		59	These fields found: [% f.value %]
		60	[% ELSIF ( f.already_in_db ) %]
		61	Patron already in database: [% f.value %]
		62	[% ELSE %]
		63	[% f.name %] : [% f.value %]
		64	[% END %]
		65	</li>
		66	[% END %]
		67	</ul>
		68	</div>
		69	[% END %]
		70
		71	[% IF ( errors ) %]
		72	<br /><br />
		73
		74	<div>
		75	<h5>Error analysis:</h5>
		76	<ul>
		77	[% FOREACH e IN errors %]
		78	[% IF ( e.badheader ) %]<li>Header row could not be parsed</li>[% END %]
		79
		80	[% FOREACH missing_critical IN e.missing_criticals %]
		81	<li class="line_error">
		82	Line <span class="linenumber">[% missing_critical.line %]</span>
		83
		84	[% IF ( missing_critical.badparse ) %]
		85	could not be parsed!
		86	[% ELSIF ( missing_critical.bad_date ) %]
		87	has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]"
		88	[% ELSE %]
		89	Critical field "[% missing_critical.key %]"
		90
		91	[% IF ( missing_critical.branch_map ) %]
		92	has unrecognized value "[% missing_critical.value %]"
		93	[% ELSIF ( missing_critical.category_map ) %]
		94	has unrecognized value "[% missing_critical.value %]"
		95	[% ELSE %]
		96	missing
		97	[% END %]
		98
		99	(borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
		100	[% END %]
		101
		102	<br/>
		103	<code>[% missing_critical.lineraw %]</code>
		104	</li>
		105	[% END %]
		106
		107	[% IF e.invalid_cardnumber %]
		108	<li class="line_error">
		109	Cardnumber [% e.cardnumber %] is not a valid cardnumber
		110	[% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %]
		111	</li>
		112	[% END %]
81	[% END %]	113	[% END %]
82	(borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).	114	</ul>
83	[% END %]	115	</div>
84	<br /><code>[% missing_critical.lineraw %]</code>
85	</li>
86	[% END %]
87	[% IF ERROR.invalid_cardnumber %]
88	<li class="line_error">
89	Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber
90	[% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %]
91	</li>
92	[% END %]
93	[% END %]	116	[% END %]
94	</ul>
95	</div>
96	[% END %]
97	[% ELSE %]	117	[% ELSE %]
98	<ul>	118	<ul>
99	<li>Select a file to import into the borrowers table</li>	119	<li>Select a file to import into the borrowers table</li>
100	<li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>	120	<li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
101	</ul>	121	</ul>
102	<form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">	122
103	<fieldset class="rows">	123	<form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
104	<legend>Import into the borrowers table</legend>	124	<fieldset class="rows">
105	<ol>	125	<legend>Import into the borrowers table</legend>
106	<li>	126
107	<label for="uploadborrowers">Select the file to import: </label>	127	<ol>
108	<input type="file" id="uploadborrowers" name="uploadborrowers" />	128	<li>
109	</li>	129	<label for="uploadborrowers">Select the file to import: </label>
110	</ol></fieldset>	130	<input type="file" id="uploadborrowers" name="uploadborrowers" />
111	<fieldset class="rows">	131	</li>
112	<legend>Field to use for record matching</legend>	132	</ol>
113	<ol>	133	</fieldset>
114	<li class="radio">	134
115	<select name="matchpoint" id="matchpoint">	135	<fieldset class="rows">
116	<option value="cardnumber">Cardnumber</option>	136	<legend>Field to use for record matching</legend>
117	<option value="userid">Username</option>	137	<ol>
118	[% FOREACH matchpoint IN matchpoints %]	138	<li class="radio">
119	<option value="[% matchpoint.code %]">[% matchpoint.description %]</option>	139	<select name="matchpoint" id="matchpoint">
		140	<option value="cardnumber">Cardnumber</option>
		141	[% FOREACH matchpoint IN matchpoints %]
		142	<option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
		143	[% END %]
		144	</select>
		145	</li>
		146	</ol>
		147	</fieldset>
		148
		149	<fieldset class="rows">
		150	<legend>Default values</legend>
		151
		152	<ol>
		153	[% FOREACH borrower_field IN borrower_fields %]
		154
		155	[% SWITCH borrower_field.field %]
		156	[% CASE 'branchcode' %]
		157	<li>
		158	<label class="description" for="branchcode">[% borrower_field.description %]: </label>
		159	<select id="branchcode" name="branchcode">
		160	<option value="" selected="selected"></option>
		161	[% FOREACH branch IN branches %]
		162	<option value="[% branch.branchcode %]">[% branch.branchname %]</option>
		163	[% END %]
		164	</select>
		165	<span class="field_hint">[% borrower_field.field %]</span>
		166	</li>
		167	[% CASE 'categorycode' %]
		168	<li>
		169	<label class="description" for="categorycode">[% borrower_field.description %]: </label>
		170	<select id="categorycode" name="categorycode">
		171	<option value="" selected="selected"></option>
		172	[% FOREACH category IN categories %]
		173	<option value="[% category.categorycode %]">[% category.description %]</option>
		174	[% END %]
		175	</select>
		176	<span class="field_hint">[% borrower_field.field %]</span>
		177	</li>
		178	[% CASE %]
		179	<li>
		180	<label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
		181	<input id="[% borrower_field.field %]" name="[% borrower_field.field %]" />
		182	<span class="field_hint">[% borrower_field.field %]</span>
		183	</li>
120	[% END %]	184	[% END %]
121	</select>	185
122	</li>	186	[% END %]
123	</ol>	187
124	</fieldset>	188	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
125	<fieldset class="rows">	189	<li>
126	<legend>Default values</legend>	190	<label class="description" for="patron_attributes">Patron attributes: </label>
127	<ol>	191	<input id="patron_attributes" name="patron_attributes" />
128	[% FOREACH borrower_field IN borrower_fields %]	192	<span class="field_hint">patron_attributes</span>
129	[% SWITCH borrower_field.field %]	193	</li>
130	[% CASE 'branchcode' %]	194	[% END %]
131	<li>	195
132	<label class="description" for="branchcode">[% borrower_field.description %]: </label>	196	</ol>
133	<select id="branchcode" name="branchcode">	197	</fieldset>
134	<option value="" selected="selected"></option>	198
135	[% FOREACH branch IN branches %]	199	<fieldset class="rows">
136	<option value="[% branch.branchcode %]">	200	<legend>If matching record is already in the borrowers table:</legend>
137	[% branch.branchname %]</option>	201
138	[% END %]	202	<ol>
139	</select><span class="field_hint">[% borrower_field.field %]</span>	203	<li class="radio">
140	</li>	204	<input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
141	[% CASE 'categorycode' %]	205	</li>
142	<li>	206
143	<label class="description" for="categorycode">[% borrower_field.description %]: </label>	207	<li class="radio">
144	<select id="categorycode" name="categorycode">	208	<input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
145	<option value="" selected="selected"></option>	209	</li>
146	[% FOREACH category IN categories %]	210	</ol>
147	<option value="[% category.categorycode %]">	211	</fieldset>
148	[% category.description %]</option>	212
		213	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
		214	<fieldset class="rows">
		215	<legend>Patron attributes</legend>
		216
		217	<ol>
		218	<li class="radio">
		219	<input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
		220	</li>
		221
		222	<li class="radio">
		223	<input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
		224	</li>
		225	</ol>
		226	</fieldset>
149	[% END %]	227	[% END %]
150	</select><span class="field_hint">[% borrower_field.field %]</span>	228
151	</li>	229	<fieldset class="action"><input type="submit" value="Import" /></fieldset>
152	[% CASE %]	230	</form>
153	<li>
154	<label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
155	<input id="[% borrower_field.field %]" name="[% borrower_field.field %]" /><span class="field_hint">[% borrower_field.field %]</span>
156	</li>
157	[% END %]
158	[% END %]
159	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
160	<li>
161	<label class="description" for="patron_attributes">Patron attributes: </label>
162	<input id="patron_attributes" name="patron_attributes" />
163	<span class="field_hint">patron_attributes</span>
164	</li>
165	[% END %]	231	[% END %]
166	</ol></fieldset>	232
167	<fieldset class="rows">	233	</div>
168	<legend>If matching record is already in the borrowers table:</legend>	234
169	<ol><li class="radio">	235	<div class="yui-u">
170	<input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>	236	<h2>Notes:</h2>
		237	<ul>
		238	<li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>
		239
		240	<li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b> Values are comma-separated.</li>
		241
		242	<li>
		243	OR choose which fields you want to supply from the following list:
		244	<ul>
		245	<li>
		246	[% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
		247	</li>
		248	</ul>
171	</li>	249	</li>
172	<li class="radio">	250
173	<input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>	251	[% IF ( ExtendedPatronAttributes ) %]
		252	<li>
		253	If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>"STARTDATE:January 1, 2010","TRACK:Day"</b>. The second syntax would be required if the data might have a comma in it, like a date string.
		254	</li>
		255	[% END %]
		256
		257	<li>
		258	The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.
174	</li>	259	</li>
175	</ol>	260
176	</fieldset>	261	<li>
177	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]	262	'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
178	<fieldset class="rows">
179	<legend>Patron attributes</legend>
180	<ol><li class="radio">
181	<input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
182	</li>	263	</li>
183	<li class="radio">	264
184	<input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>	265	<li>
		266	Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'. Alternatively,
		267	you can supply dates in ISO format (e.g., '2010-10-28').
185	</li>	268	</li>
186	</ol>	269	</ul>
187	</fieldset>	270
188	[% END %]
189	<fieldset class="action"><input type="submit" value="Import" /></fieldset>
190	</form>
191	[% END %]
192	</div>	271	</div>
193	<div class="yui-u">	272	</div>
194	<h2>Notes:</h2>	273	</div>
195	<ul>	274	</div>
196	<li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>	275
197	<li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b> Values are comma-separated.</li>	276	<div class="yui-b noprint">
198	<li>OR choose which fields you want to supply from the following list:<ul><li>	277	[% INCLUDE 'tools-menu.inc' %]
199	[% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]	278	</div>
200	</li></ul></li>	279	</div>
201	[% IF ( ExtendedPatronAttributes ) %]	280
202	<li>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>"STARTDATE:January 1, 2010","TRACK:Day"</b>. The second syntax would be required if the data might have a comma in it, like a date string.
203	</li>
204	[% END %]
205	<li>The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.</li>
206	<li>'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).</li>
207	<li>Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'. Alternatively,
208	you can supply dates in ISO format (e.g., '2010-10-28').
209	</li>
210	</ul>
211
212	</div>
213	</div>
214	</div>
215	</div>
216	<div class="yui-b noprint">
217	[% INCLUDE 'tools-menu.inc' %]
218	</div>
219	</div>
220	[% INCLUDE 'intranet-bottom.inc' %]	281	[% INCLUDE 'intranet-bottom.inc' %]

Lines 34-54 Link Here

(-)a/tools/import_borrowers.pl (-306 / +40 lines)
34	# dates should be in the format you have set up Koha to expect	34	# dates should be in the format you have set up Koha to expect
35	# branchcode and categorycode need to be valid	35	# branchcode and categorycode need to be valid
36		36
37	use strict;	37	use Modern::Perl;
38	use warnings;
39		38
40	use C4::Auth;	39	use C4::Auth;
41	use C4::Output;	40	use C4::Output;
42	use C4::Context;
43	use C4::Branch qw/GetBranchesLoop GetBranchName/;
44	use C4::Members;
45	use C4::Members::Attributes qw(:all);
46	use C4::Members::AttributeTypes;
47	use C4::Members::Messaging;
48	use C4::Reports::Guided;
49	use C4::Templates;	41	use C4::Templates;
50	use Koha::Borrower::Debarments;	42
51	use Koha::DateUtils;	43	use C4::Branch;
		44	use C4::Members;
		45
		46	use Koha::Patrons::Import qw(import_patrons);
52		47
53	use Text::CSV;	48	use Text::CSV;
54		49
Lines 61-76 use CGI qw ( -utf8 ); Link Here
61	# use encoding 'utf8'; # don't do this	56	# use encoding 'utf8'; # don't do this
62		57
63	my ( @errors, @feedback );	58	my ( @errors, @feedback );
64	my $extended = C4::Context->preference('ExtendedPatronAttributes');	59	my $extended = C4::Context->preference('ExtendedPatronAttributes');
65	my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');	60
66	my @columnkeys = C4::Members::columns();	61	my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
67	@columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } @columnkeys;	62	push( @columnkeys, 'patron_attributes' ) if $extended;
68	if ($extended) {
69	push @columnkeys, 'patron_attributes';
70	}
71		63
72	my $input = CGI->new();	64	my $input = CGI->new();
73	our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode
74		65
75	#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX	66	#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
76		67
Lines 97-102 $columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ]; Link Here
97	$template->param( borrower_fields => $columns );	88	$template->param( borrower_fields => $columns );
98		89
99	if ( $input->param('sample') ) {	90	if ( $input->param('sample') ) {
		91	our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode
100	print $input->header(	92	print $input->header(
101	-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?	93	-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
102	-attachment => 'patron_import.csv',	94	-attachment => 'patron_import.csv',
Lines 105-413 if ( $input->param('sample') ) { Link Here
105	print $csv->string, "\n";	97	print $csv->string, "\n";
106	exit 1;	98	exit 1;
107	}	99	}
		100
108	my $uploadborrowers = $input->param('uploadborrowers');	101	my $uploadborrowers = $input->param('uploadborrowers');
109	my $matchpoint = $input->param('matchpoint');	102	my $matchpoint = $input->param('matchpoint');
110	if ($matchpoint) {	103	if ($matchpoint) {
111	$matchpoint =~ s/^patron_attribute_//;	104	$matchpoint =~ s/^patron_attribute_//;
112	}	105	}
113	my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
114		106
115	$template->param( SCRIPT_NAME => '/cgi-bin/koha/tools/import_borrowers.pl' );	107	$template->param( SCRIPT_NAME => '/cgi-bin/koha/tools/import_borrowers.pl' );
116		108
117	if ( $uploadborrowers && length($uploadborrowers) > 0 ) {	109	if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
118	push @feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };	110	my $handle = $input->upload('uploadborrowers');
119	my $handle = $input->upload('uploadborrowers');
120	my $uploadinfo = $input->uploadInfo($uploadborrowers);
121	foreach ( keys %$uploadinfo ) {
122	push @feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
123	}
124	my $imported = 0;
125	my $alreadyindb = 0;
126	my $overwritten = 0;
127	my $invalid = 0;
128	my $matchpoint_attr_type;
129	my %defaults = $input->Vars;	111	my %defaults = $input->Vars;
130		112
131	# use header line to construct key to column map	113	my $return = Koha::Patrons::Import::import_patrons(
132	my $borrowerline = <$handle>;	114	{
133	my $status = $csv->parse($borrowerline);	115	file => $handle,
134	($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline };	116	defaults => \%defaults,
135	my @csvcolumns = $csv->fields();	117	matchpoint => $matchpoint,
136	my %csvkeycol;	118	overwrite_cardnumber => $input->param('overwrite_cardnumber'),
137	my $col = 0;	119	preserve_extended_attributes => $input->param('ext_preserve') \|\| 0,
138	foreach my $keycol (@csvcolumns) {
139
140	# columnkeys don't contain whitespace, but some stupid tools add it
141	$keycol =~ s/ +//g;
142	$csvkeycol{$keycol} = $col++;
143	}
144
145	#warn($borrowerline);
146	my $ext_preserve = $input->param('ext_preserve') \|\| 0;
147	if ($extended) {
148	$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
149	}
150
151	push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
152	my $today_iso = output_pref( { dt => dt_from_string, dateonly => 1, dateformat => 'iso' } );
153	my @criticals = qw(surname branchcode categorycode); # there probably should be others
154	my @bad_dates; # I've had a few.
155	LINE: while ( my $borrowerline = <$handle> ) {
156	my %borrower;
157	my @missing_criticals;
158	my $patron_attributes;
159	my $status = $csv->parse($borrowerline);
160	my @columns = $csv->fields();
161	if ( !$status ) {
162	push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
163	}
164	elsif ( @columns == @columnkeys ) {
165	@borrower{@columnkeys} = @columns;
166
167	# MJR: try to fill blanks gracefully by using default values
168	foreach my $key (@columnkeys) {
169	if ( $borrower{$key} !~ /\S/ ) {
170	$borrower{$key} = $defaults{$key};
171	}
172	}
173	}
174	else {
175	# MJR: try to recover gracefully by using default values
176	foreach my $key (@columnkeys) {
177	if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
178	$borrower{$key} = $columns[ $csvkeycol{$key} ];
179	}
180	elsif ( $defaults{$key} ) {
181	$borrower{$key} = $defaults{$key};
182	}
183	elsif ( scalar grep { $key eq $_ } @criticals ) {
184
185	# a critical field is undefined
186	push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
187	}
188	else {
189	$borrower{$key} = '';
190	}
191	}
192	}
193
194	#warn join(':',%borrower);
195	if ( $borrower{categorycode} ) {
196	push @missing_criticals,
197	{
198	key => 'categorycode',
199	line => $.,
200	lineraw => $borrowerline,
201	value => $borrower{categorycode},
202	category_map => 1
203	}
204	unless GetBorrowercategory( $borrower{categorycode} );
205	}
206	else {
207	push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
208	}
209	if ( $borrower{branchcode} ) {
210	push @missing_criticals,
211	{
212	key => 'branchcode',
213	line => $.,
214	lineraw => $borrowerline,
215	value => $borrower{branchcode},
216	branch_map => 1
217	}
218	unless GetBranchName( $borrower{branchcode} );
219	}
220	else {
221	push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
222	}
223	if (@missing_criticals) {
224	foreach (@missing_criticals) {
225	$_->{borrowernumber} = $borrower{borrowernumber} \|\| 'UNDEF';
226	$_->{surname} = $borrower{surname} \|\| 'UNDEF';
227	}
228	$invalid++;
229	( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };
230
231	# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance.
232	next LINE;
233	}
234	if ($extended) {
235	my $attr_str = $borrower{patron_attributes};
236	$attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes
237	$attr_str =~ s/\xe2\x80\x9d/"/g;
238	push @feedback,
239	{ feedback => 1, name => 'attribute string', value => $attr_str, filename => $uploadborrowers };
240	delete $borrower{patron_attributes}
241	; # not really a field in borrowers, so we don't want to pass it to ModMember.
242	$patron_attributes = extended_attributes_code_value_arrayref($attr_str);
243	}
244
245	# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
246	foreach (qw(dateofbirth dateenrolled dateexpiry)) {
247	my $tempdate = $borrower{$_} or next;
248	$tempdate =
249	eval { output_pref( { dt => dt_from_string($tempdate), dateonly => 1, dateformat => 'iso' } ); };
250	if ($tempdate) {
251	$borrower{$_} = $tempdate;
252	}
253	else {
254	$borrower{$_} = '';
255	push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
256	}
257	}	120	}
258	$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};	121	);
259	$borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
260	unless $borrower{dateexpiry};
261	my $borrowernumber;
262	my $member;
263	if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
264	$member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
265	if ($member) {
266	$borrowernumber = $member->{'borrowernumber'};
267	}
268	}
269	elsif ( ( $matchpoint eq 'userid' ) && ( $borrower{'userid'} ) ) {
270	$member = GetMember( 'userid' => $borrower{'userid'} );
271	if ($member) {
272	$borrowernumber = $member->{'borrowernumber'};
273	}
274	}
275	elsif ($extended) {
276	if ( defined($matchpoint_attr_type) ) {
277	foreach my $attr (@$patron_attributes) {
278	if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
279	my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
280	$borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
281	last;
282	}
283	}
284	}
285	}
286
287	if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
288	push @errors,
289	{
290	invalid_cardnumber => 1,
291	borrowernumber => $borrowernumber,
292	cardnumber => $borrower{cardnumber}
293	};
294	$invalid++;
295	next;
296	}
297
298	if ($borrowernumber) {
299
300	# borrower exists
301	unless ($overwrite_cardnumber) {
302	$alreadyindb++;
303	$template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber );
304	next LINE;
305	}
306	$borrower{'borrowernumber'} = $borrowernumber;
307	for my $col ( keys %borrower ) {
308
309	# use values from extant patron unless our csv file includes this column or we provided a default.
310	# FIXME : You cannot update a field with a perl-evaluated false value using the defaults.
311
312	# The password is always encrypted, skip it!
313	next if $col eq 'password';
314
315	unless ( exists( $csvkeycol{$col} ) \|\| $defaults{$col} ) {
316	$borrower{$col} = $member->{$col} if ( $member->{$col} );
317	}
318	}
319	unless ( ModMember(%borrower) ) {
320	$invalid++;
321
322	# until we have better error trapping, we have no way of knowing why ModMember errored out...
323	push @errors, { unknown_error => 1 };
324	$template->param( 'lastinvalid' => $borrower{'surname'} . ' / ' . $borrowernumber );
325	next LINE;
326	}
327	if ( $borrower{debarred} ) {
328
329	# Check to see if this debarment already exists
330	my $debarrments = GetDebarments(
331	{
332	borrowernumber => $borrowernumber,
333	expiration => $borrower{debarred},
334	comment => $borrower{debarredcomment}
335	}
336	);
337
338	# If it doesn't, then add it!
339	unless (@$debarrments) {
340	AddDebarment(
341	{
342	borrowernumber => $borrowernumber,
343	expiration => $borrower{debarred},
344	comment => $borrower{debarredcomment}
345	}
346	);
347	}
348	}
349	if ($extended) {
350	if ($ext_preserve) {
351	my $old_attributes = GetBorrowerAttributes($borrowernumber);
352	$patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes )
353	; #TODO: expose repeatable options in template
354	}
355	push @errors, { unknown_error => 1 }
356	unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
357	}
358	$overwritten++;
359	$template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber );
360	}
361	else {
362	# FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
363	# At least this is closer to AddMember than in members/memberentry.pl
364	if ( !$borrower{'cardnumber'} ) {
365	$borrower{'cardnumber'} = fixup_cardnumber(undef);
366	}
367	if ( $borrowernumber = AddMember(%borrower) ) {
368		122
369	if ( $borrower{debarred} ) {	123	my $feedback = $return->{feedback};
370	AddDebarment(	124	my $errors = $return->{errors};
371	{	125	my $imported = $return->{imported};
372	borrowernumber => $borrowernumber,	126	my $overwritten = $return->{overwritten};
373	expiration => $borrower{debarred},	127	my $alreadyindb = $return->{already_in_db};
374	comment => $borrower{debarredcomment}	128	my $invalid = $return->{invalid};
375	}
376	);
377	}
378		129
379	if ($extended) {	130	my $uploadinfo = $input->uploadInfo($uploadborrowers);
380	SetBorrowerAttributes( $borrowernumber, $patron_attributes );	131	foreach ( keys %$uploadinfo ) {
381	}	132	push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
		133	}
382		134
383	if ($set_messaging_prefs) {	135	push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };
384	C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
385	{
386	borrowernumber => $borrowernumber,
387	categorycode => $borrower{categorycode}
388	}
389	);
390	}
391		136
392	$imported++;
393	$template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber );
394	}
395	else {
396	$invalid++;
397	push @errors, { unknown_error => 1 };
398	$template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember' );
399	}
400	}
401	}
402	(@errors) and $template->param( ERRORS => \@errors );
403	(@feedback) and $template->param( FEEDBACK => \@feedback );
404	$template->param(	137	$template->param(
405	'uploadborrowers' => 1,	138	uploadborrowers => 1,
406	'imported' => $imported,	139	errors => $errors,
407	'overwritten' => $overwritten,	140	feedback => $feedback,
408	'alreadyindb' => $alreadyindb,	141	imported => $imported,
409	'invalid' => $invalid,	142	overwritten => $overwritten,
410	'total' => $imported + $alreadyindb + $invalid + $overwritten,	143	alreadyindb => $alreadyindb,
		144	invalid => $invalid,
		145	total => $imported + $alreadyindb + $invalid + $overwritten,
411	);	146	);
412		147
413	}	148	}
414	-