Attachment #36694 for bug #12598

View | Details | Raw Unified | Return to bug 12598
Collapse All | Expand All




package Koha::Patrons::Import;

# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it under the
# terms of the GNU General Public License as published by the Free Software
# Foundation; either version 3 of the License, or (at your option) any later
# version.
#
# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with Koha; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

use Modern::Perl;

use Carp;
use Text::CSV;

use C4::Members;
use C4::Branch;

sub import_patrons {
    my ($params) = @_;

    my $handle               = $params->{file};
    my $matchpoint           = $params->{matchpoint};
    my $defaults             = $params->{defaults};
    my $ext_preserve         = $params->{preserve_extended_attributes};
    my $overwrite_cardnumber = $params->{overwrite_cardnumber};

    carp("No file handle passed in!") && return unless $handle;

    my $extended            = C4::Context->preference('ExtendedPatronAttributes');
    my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');

    my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
    push( @columnkeys, 'patron_attributes' ) if $extended;

    our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode

    my @feedback;
    my @errors;

    my $imported    = 0;
    my $alreadyindb = 0;
    my $overwritten = 0;
    my $invalid     = 0;
    my $matchpoint_attr_type;

    # use header line to construct key to column map
    my $borrowerline = <$handle>;
    my $status       = $csv->parse($borrowerline);
    ($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline };
    my @csvcolumns = $csv->fields();
    my %csvkeycol;
    my $col = 0;
    foreach my $keycol (@csvcolumns) {

        # columnkeys don't contain whitespace, but some stupid tools add it
        $keycol =~ s/ +//g;
        $csvkeycol{$keycol} = $col++;
    }

    #warn($borrowerline);
    if ($extended) {
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
    }

    push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
    my $today_iso = C4::Dates->new()->output('iso');
    my @criticals = qw(surname branchcode categorycode);    # there probably should be others
    my @bad_dates;                                          # I've had a few.
    my $date_re = C4::Dates->new->regexp('syspref');
    my $iso_re  = C4::Dates->new->regexp('iso');
  LINE: while ( my $borrowerline = <$handle> ) {
        my %borrower;
        my @missing_criticals;
        my $patron_attributes;
        my $status  = $csv->parse($borrowerline);
        my @columns = $csv->fields();
        if ( !$status ) {
            push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
        }
        elsif ( @columns == @columnkeys ) {
            @borrower{@columnkeys} = @columns;

            # MJR: try to fill blanks gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( $borrower{$key} !~ /\S/ ) {
                    $borrower{$key} = $defaults->{$key};
                }
            }
        }
        else {
            # MJR: try to recover gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
                    $borrower{$key} = $columns[ $csvkeycol{$key} ];
                }
                elsif ( $defaults->{$key} ) {
                    $borrower{$key} = $defaults->{$key};
                }
                elsif ( scalar grep { $key eq $_ } @criticals ) {

                    # a critical field is undefined
                    push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
                }
                else {
                    $borrower{$key} = '';
                }
            }
        }

        #warn join(':',%borrower);
        if ( $borrower{categorycode} ) {
            push @missing_criticals,
              {
                key          => 'categorycode',
                line         => $.,
                lineraw      => $borrowerline,
                value        => $borrower{categorycode},
                category_map => 1
              }
              unless GetBorrowercategory( $borrower{categorycode} );
        }
        else {
            push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
        }
        if ( $borrower{branchcode} ) {
            push @missing_criticals,
              {
                key        => 'branchcode',
                line       => $.,
                lineraw    => $borrowerline,
                value      => $borrower{branchcode},
                branch_map => 1
              }
              unless GetBranchName( $borrower{branchcode} );
        }
        else {
            push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
        }
        if (@missing_criticals) {
            foreach (@missing_criticals) {
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
                $_->{surname}        = $borrower{surname}        || 'UNDEF';
            }
            $invalid++;
            ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };

            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
            next LINE;
        }
        if ($extended) {
            my $attr_str = $borrower{patron_attributes};
            $attr_str =~ s/\xe2\x80\x9c/"/g;    # fixup double quotes in case we are passed smart quotes
            $attr_str =~ s/\xe2\x80\x9d/"/g;
            push @feedback, { feedback => 1, name => 'attribute string', value => $attr_str };
            delete $borrower{patron_attributes}
              ;    # not really a field in borrowers, so we don't want to pass it to ModMember.
            $patron_attributes = extended_attributes_code_value_arrayref($attr_str);
        }

        # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
        foreach (qw(dateofbirth dateenrolled dateexpiry)) {
            my $tempdate = $borrower{$_} or next;
            if ( $tempdate =~ /$date_re/ ) {
                $borrower{$_} = format_date_in_iso($tempdate);
            }
            elsif ( $tempdate =~ /$iso_re/ ) {
                $borrower{$_} = $tempdate;
            }
            else {
                $borrower{$_} = '';
                push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
            }
        }
        $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
        $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
          unless $borrower{dateexpiry};
        my $borrowernumber;
        my $member;
        if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
            $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
            if ($member) {
                $borrowernumber = $member->{'borrowernumber'};
            }
        }
        elsif ($extended) {
            if ( defined($matchpoint_attr_type) ) {
                foreach my $attr (@$patron_attributes) {
                    if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
                        last;
                    }
                }
            }
        }

        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
            push @errors,
              {
                invalid_cardnumber => 1,
                borrowernumber     => $borrowernumber,
                cardnumber         => $borrower{cardnumber}
              };
            $invalid++;
            next;
        }

        if ($borrowernumber) {

            # borrower exists
            unless ($overwrite_cardnumber) {
                $alreadyindb++;
                push(
                    @feedback,
                    {
                        already_in_db => 1,
                        value         => $borrower{'surname'} . ' / ' . $borrowernumber
                    }
                );
                next LINE;
            }
            $borrower{'borrowernumber'} = $borrowernumber;
            for my $col ( keys %borrower ) {

                # use values from extant patron unless our csv file includes this column or we provided a default.
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.

                # The password is always encrypted, skip it!
                next if $col eq 'password';

                unless ( exists( $csvkeycol{$col} ) || $defaults->{$col} ) {
                    $borrower{$col} = $member->{$col} if ( $member->{$col} );
                }
            }
            unless ( ModMember(%borrower) ) {
                $invalid++;

                push(
                    @errors,
                    {
                        name  => 'lastinvalid',
                        value => $borrower{'surname'} . ' / ' . $borrowernumber
                    }
                );
                next LINE;
            }
            if ( $borrower{debarred} ) {

                # Check to see if this debarment already exists
                my $debarrments = GetDebarments(
                    {
                        borrowernumber => $borrowernumber,
                        expiration     => $borrower{debarred},
                        comment        => $borrower{debarredcomment}
                    }
                );

                # If it doesn't, then add it!
                unless (@$debarrments) {
                    AddDebarment(
                        {
                            borrowernumber => $borrowernumber,
                            expiration     => $borrower{debarred},
                            comment        => $borrower{debarredcomment}
                        }
                    );
                }
            }
            if ($extended) {
                if ($ext_preserve) {
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
                    $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes );
                }
                push @errors, { unknown_error => 1 }
                  unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
            }
            $overwritten++;
            push(
                @feedback,
                {
                    feedback => 1,
                    name     => 'lastoverwritten',
                    value    => $borrower{'surname'} . ' / ' . $borrowernumber
                }
            );
        }
        else {
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
            # At least this is closer to AddMember than in members/memberentry.pl
            if ( !$borrower{'cardnumber'} ) {
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
            }
            if ( $borrowernumber = AddMember(%borrower) ) {

                if ( $borrower{debarred} ) {
                    AddDebarment(
                        {
                            borrowernumber => $borrowernumber,
                            expiration     => $borrower{debarred},
                            comment        => $borrower{debarredcomment}
                        }
                    );
                }

                if ($extended) {
                    SetBorrowerAttributes( $borrowernumber, $patron_attributes );
                }

                if ($set_messaging_prefs) {
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
                        {
                            borrowernumber => $borrowernumber,
                            categorycode   => $borrower{categorycode}
                        }
                    );
                }

                $imported++;
                push(
                    @feedback,
                    {
                        feedback => 1,
                        name     => 'lastimported',
                        value    => $borrower{'surname'} . ' / ' . $borrowernumber
                    }
                );
            }
            else {
                $invalid++;
                push @errors, { unknown_error => 1 };
                push(
                    @errors,
                    {
                        name  => 'lastinvalid',
                        value => $borrower{'surname'} . ' / AddMember',
                    }
                );
            }
        }
    }

    return {
        feedback      => \@feedback,
        errors        => \@errors,
        imported      => $imported,
        overwritten   => $overwritten,
        already_in_db => $alreadyindb,
        invalid       => $invalid,
    };
}

END { }    # module clean-up code here (global destructor)

1;

__END__

=head1 AUTHOR

Koha Team

=cut




     <div class="yui-u first">
<h1>Import patrons</h1>
[% IF ( uploadborrowers ) %]
    <h5>Import results :</h5>
<ul>
	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
	<li>[% total %] records parsed</li>
	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
</ul>
  [% IF ( FEEDBACK ) %]
  <br /><br />
    <div>
    <h5>Feedback:</h5>
    <ul class="feedback">
    [% FOREACH FEEDBAC IN FEEDBACK %]
    <li>
        [% IF ( FEEDBAC.filename ) %]Parsing upload file <span class="filename">[% FEEDBAC.filename %]</span>
        [% ELSIF ( FEEDBAC.backend ) %]Upload parsed using [% FEEDBAC.backend %]
        [% ELSIF ( FEEDBAC.headerrow ) %]These fields found: [% FEEDBAC.value %]
        [% ELSE %][% FEEDBAC.name %] : [% FEEDBAC.value %]
        [% END %]
    </li>
    [% END %]
    </ul>
    </div>
  [% END %]
  [% IF ( ERRORS ) %]
  <br /><br />
    <div>
    <h5>Error analysis:</h5>
    <ul>
        <li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
        <li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
        <li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
        <li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
        <li>[% total %] records parsed</li>
        <li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
    </ul>

    [% IF ( feedback ) %]
        <br /><br />

        <div>
            <h5>Feedback:</h5>
                <ul class="feedback">
                    [% FOREACH f IN feedback %]
                        <li>
                            [% IF ( f.filename ) %]
                                Parsing upload file <span class="filename">[% f.filename %]</span>
                            [% ELSIF ( f.backend ) %]
                                Upload parsed using [% f.backend %]
                            [% ELSIF ( f.headerrow ) %]
                                These fields found: [% f.value %]
                            [% ELSIF ( f.already_in_db ) %]
                                Patron already in database: [% f.value %]
                            [% ELSE %]
                                [% f.name %] : [% f.value %]
                            [% END %]
                        </li>
                    [% END %]
                </ul>
        </div>
    [% END %]

    [% IF ( errors ) %]
        <br /><br />

        <div>
            <h5>Error analysis:</h5>
            <ul>
                [% FOREACH e IN errors %]
                    [% IF ( e.badheader ) %]<li>Header row could not be parsed</li>[% END %]

                    [% FOREACH missing_critical IN e.missing_criticals %]
                        <li class="line_error">
                            Line <span class="linenumber">[% missing_critical.line %]</span>

                            [% IF ( missing_critical.badparse ) %]
                                could not be parsed!
                            [% ELSIF ( missing_critical.bad_date ) %]
                                has &quot;[% missing_critical.key %]&quot; in unrecognized format: &quot;[% missing_critical.value %]&quot;
                            [% ELSE %]
                                Critical field &quot;[% missing_critical.key %]&quot;

                                [% IF ( missing_critical.branch_map ) %]
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
                                [% ELSIF ( missing_critical.category_map ) %]
                                    has unrecognized value &quot;[% missing_critical.value %]&quot;
                                [% ELSE %]
                                    missing
                                [% END %]

                                (borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
                            [% END %]

                            <br/>
                            <code>[% missing_critical.lineraw %]</code>
                        </li>
                    [% END %]

                    [% IF e.invalid_cardnumber %]
                        <li class="line_error">
                            Cardnumber [% e.cardnumber %] is not a valid cardnumber
                            [% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %]
                        </li>
                    [% END %]
                [% END %]
            </ul>
        </div>
            <br /><code>[% missing_critical.lineraw %]</code>
        </li>
        [% END %]
        [% IF ERROR.invalid_cardnumber %]
            <li class="line_error">
                Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber
                [% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %]
            </li>
        [% END %]
    [% END %]
    </ul>
    </div>
  [% END %]
[% ELSE %]
    <ul>
        <li>Select a file to import into the borrowers table</li>
        <li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
    </ul>

    <form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
        <fieldset class="rows">
            <legend>Import into the borrowers table</legend>

            <ol>
                <li>
                    <label for="uploadborrowers">Select the file to import: </label>
                    <input type="file" id="uploadborrowers" name="uploadborrowers" />
                </li>
            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>Field to use for record matching</legend>
            <ol>
                <li class="radio">
                    <select name="matchpoint" id="matchpoint">
                        <option value="cardnumber">Cardnumber</option>
                        [% FOREACH matchpoint IN matchpoints %]
                            <option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
                        [% END %]
                    </select>
                </li>
            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>Default values</legend>

            <ol>
                [% FOREACH borrower_field IN borrower_fields %]

                    [% SWITCH borrower_field.field %]
                    [% CASE 'branchcode' %]
                        <li>
                            <label class="description" for="branchcode">[% borrower_field.description %]: </label>
                            <select id="branchcode" name="branchcode">
                                <option value="" selected="selected"></option>
                                [% FOREACH branch IN branches %]
                                    <option value="[% branch.branchcode %]">[% branch.branchname %]</option>
                                [% END %]
                            </select>
                            <span class="field_hint">[% borrower_field.field %]</span>
                        </li>
                    [% CASE 'categorycode' %]
                        <li>
                            <label class="description" for="categorycode">[% borrower_field.description %]: </label>
                            <select id="categorycode" name="categorycode">
                                <option value="" selected="selected"></option>
                                [% FOREACH category IN categories %]
                                    <option value="[% category.categorycode %]">[% category.description %]</option>
                                [% END %]
                            </select>
                            <span class="field_hint">[% borrower_field.field %]</span>
                        </li>
                    [% CASE %]
                        <li>
                            <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
                            <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" />
                            <span class="field_hint">[% borrower_field.field %]</span>
                        </li>
                    [% END %]

                [% END %]

                [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
                    <li>
                        <label class="description" for="patron_attributes">Patron attributes: </label>
                        <input id="patron_attributes" name="patron_attributes" />
                        <span class="field_hint">patron_attributes</span>
                    </li>
                [% END %]

            </ol>
        </fieldset>

        <fieldset class="rows">
            <legend>If matching record is already in the borrowers table:</legend>

            <ol>
                <li class="radio">
                    <input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
                </li>

                <li class="radio">
                    <input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
                </li>
            </ol>
        </fieldset>

        [% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
            <fieldset class="rows">
                <legend>Patron attributes</legend>

                <ol>
                    <li class="radio">
                        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
                    </li>

                    <li class="radio">
                        <input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
                    </li>
                </ol>
            </fieldset>
        [% END %]

        <fieldset class="action"><input type="submit" value="Import" /></fieldset>
    </form>
    <li>
        <label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
        <input id="[% borrower_field.field %]" name="[% borrower_field.field %]" /><span class="field_hint">[% borrower_field.field %]</span>
    </li>
  [% END %]
[% END %]
[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
    <li>
        <label class="description" for="patron_attributes">Patron attributes: </label>
        <input id="patron_attributes" name="patron_attributes" />
        <span class="field_hint">patron_attributes</span>
    </li>
[% END %]

</div>

<div class="yui-u">
    <h2>Notes:</h2>
    <ul>
        <li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>

        <li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b>  Values are comma-separated.</li>

        <li>
            OR choose which fields you want to supply from the following list:
            <ul>
                <li>
                    [% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
                </li>
            </ul>
        </li>

        [% IF ( ExtendedPatronAttributes ) %]
            <li>
                If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>&quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</b>.  The second syntax would be required if the data might have a comma in it, like a date string.
            </li>
        [% END %]

        <li>
            The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.
        </li>

        <li>
            'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
	<fieldset class="rows">
    <legend>Patron attributes</legend>
    <ol><li class="radio">
        <input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
        </li>

        <li>
            Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.  Alternatively,
you can supply dates in ISO format (e.g., '2010-10-28').
        </li>
    </ul>

    [% END %]
	<fieldset class="action"><input type="submit" value="Import" /></fieldset>
</form>
[% END %]
</div>
</div>
</div>
</div>

<div class="yui-b noprint">
    [% INCLUDE 'tools-menu.inc' %]
</div>
</div>

<li>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>&quot;STARTDATE:January 1, 2010&quot;,&quot;TRACK:Day&quot;</b>.  The second syntax would be required if the data might have a comma in it, like a date string.
</li>
[% END %]
<li>The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.</li>
<li>'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).</li>
<li>Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'.  Alternatively,
you can supply dates in ISO format (e.g., '2010-10-28').
</li>
</ul>

     </div>
    </div>
   </div>
  </div>
  <div class="yui-b noprint">
[% INCLUDE 'tools-menu.inc' %]
  </div>
 </div>
[% INCLUDE 'intranet-bottom.inc' %]




# dates should be in the format you have set up Koha to expect
# branchcode and categorycode need to be valid

use Modern::Perl;
use warnings;

use C4::Auth;
use C4::Output;
use C4::Dates qw(format_date_in_iso);
use C4::Context;
use C4::Branch qw/GetBranchesLoop GetBranchName/;
use C4::Members;
use C4::Members::Attributes qw(:all);
use C4::Members::AttributeTypes;
use C4::Members::Messaging;
use C4::Reports::Guided;
use C4::Templates;

use C4::Branch;
use C4::Members;

use Koha::Patrons::Import qw(import_patrons);

use Text::CSV;


# use encoding 'utf8';    # don't do this

my ( @errors, @feedback );
my $extended = C4::Context->preference('ExtendedPatronAttributes');

my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
push( @columnkeys, 'patron_attributes' ) if $extended;
if ($extended) {
    push @columnkeys, 'patron_attributes';
}

my $input = CGI->new();
our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode

#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX


$template->param( borrower_fields => $columns );

if ( $input->param('sample') ) {
    our $csv = Text::CSV->new( { binary => 1 } );    # binary needed for non-ASCII Unicode
    print $input->header(
        -type       => 'application/vnd.sun.xml.calc',    # 'application/vnd.ms-excel' ?
        -attachment => 'patron_import.csv',

    print $csv->string, "\n";
    exit 1;
}

my $uploadborrowers = $input->param('uploadborrowers');
my $matchpoint      = $input->param('matchpoint');
if ($matchpoint) {
    $matchpoint =~ s/^patron_attribute_//;
}
my $overwrite_cardnumber = $input->param('overwrite_cardnumber');

$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );

if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
    my $handle   = $input->upload('uploadborrowers');
    my $handle     = $input->upload('uploadborrowers');
    my $uploadinfo = $input->uploadInfo($uploadborrowers);
    foreach ( keys %$uploadinfo ) {
        push @feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
    }
    my $imported    = 0;
    my $alreadyindb = 0;
    my $overwritten = 0;
    my $invalid     = 0;
    my $matchpoint_attr_type;
    my %defaults = $input->Vars;

    my $return = Koha::Patrons::Import::import_patrons(
        {
            file                         => $handle,
            defaults                     => \%defaults,
            matchpoint                   => $matchpoint,
            overwrite_cardnumber         => $input->param('overwrite_cardnumber'),
            preserve_extended_attributes => $input->param('ext_preserve') || 0,
    foreach my $keycol (@csvcolumns) {

        # columnkeys don't contain whitespace, but some stupid tools add it
        $keycol =~ s/ +//g;
        $csvkeycol{$keycol} = $col++;
    }

    #warn($borrowerline);
    my $ext_preserve = $input->param('ext_preserve') || 0;
    if ($extended) {
        $matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
    }

    push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
    my $today_iso = C4::Dates->new()->output('iso');
    my @criticals = qw(surname branchcode categorycode);    # there probably should be others
    my @bad_dates;                                          # I've had a few.
    my $date_re = C4::Dates->new->regexp('syspref');
    my $iso_re  = C4::Dates->new->regexp('iso');
  LINE: while ( my $borrowerline = <$handle> ) {
        my %borrower;
        my @missing_criticals;
        my $patron_attributes;
        my $status  = $csv->parse($borrowerline);
        my @columns = $csv->fields();
        if ( !$status ) {
            push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
        }
    );
            @borrower{@columnkeys} = @columns;

            # MJR: try to fill blanks gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( $borrower{$key} !~ /\S/ ) {
                    $borrower{$key} = $defaults{$key};
                }
            }
        }
        else {
            # MJR: try to recover gracefully by using default values
            foreach my $key (@columnkeys) {
                if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
                    $borrower{$key} = $columns[ $csvkeycol{$key} ];
                }
                elsif ( $defaults{$key} ) {
                    $borrower{$key} = $defaults{$key};
                }
                elsif ( scalar grep { $key eq $_ } @criticals ) {

                    # a critical field is undefined
                    push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
                }
                else {
                    $borrower{$key} = '';
                }
            }
        }

        #warn join(':',%borrower);
        if ( $borrower{categorycode} ) {
            push @missing_criticals,
              {
                key          => 'categorycode',
                line         => $.,
                lineraw      => $borrowerline,
                value        => $borrower{categorycode},
                category_map => 1
              }
              unless GetBorrowercategory( $borrower{categorycode} );
        }
        else {
            push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
        }
        if ( $borrower{branchcode} ) {
            push @missing_criticals,
              {
                key        => 'branchcode',
                line       => $.,
                lineraw    => $borrowerline,
                value      => $borrower{branchcode},
                branch_map => 1
              }
              unless GetBranchName( $borrower{branchcode} );
        }
        else {
            push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
        }
        if (@missing_criticals) {
            foreach (@missing_criticals) {
                $_->{borrowernumber} = $borrower{borrowernumber} || 'UNDEF';
                $_->{surname}        = $borrower{surname}        || 'UNDEF';
            }
            $invalid++;
            ( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };

            # The first 25 errors are enough.  Keeping track of 30,000+ would destroy performance.
            next LINE;
        }
        if ($extended) {
            my $attr_str = $borrower{patron_attributes};
            $attr_str =~ s/\xe2\x80\x9c/"/g;    # fixup double quotes in case we are passed smart quotes
            $attr_str =~ s/\xe2\x80\x9d/"/g;
            push @feedback,
              { feedback => 1, name => 'attribute string', value => $attr_str, filename => $uploadborrowers };
            delete $borrower{patron_attributes}
              ;    # not really a field in borrowers, so we don't want to pass it to ModMember.
            $patron_attributes = extended_attributes_code_value_arrayref($attr_str);
        }

        # Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
        foreach (qw(dateofbirth dateenrolled dateexpiry)) {
            my $tempdate = $borrower{$_} or next;
            if ( $tempdate =~ /$date_re/ ) {
                $borrower{$_} = format_date_in_iso($tempdate);
            }
            elsif ( $tempdate =~ /$iso_re/ ) {
                $borrower{$_} = $tempdate;
            }
            else {
                $borrower{$_} = '';
                push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
            }
        }
        $borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
        $borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
          unless $borrower{dateexpiry};
        my $borrowernumber;
        my $member;
        if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
            $member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
            if ($member) {
                $borrowernumber = $member->{'borrowernumber'};
            }
        } elsif ( ($matchpoint eq 'userid') && ($borrower{'userid'}) ) {
            $member = GetMember( 'userid' => $borrower{'userid'} );
            if ($member) {
                $borrowernumber = $member->{'borrowernumber'};
            }
        } elsif ($extended) {
            if (defined($matchpoint_attr_type)) {
                foreach my $attr (@$patron_attributes) {
                    if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
                        my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
                        $borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
                        last;
                    }
                }
            }
        }

        if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
            push @errors,
              {
                invalid_cardnumber => 1,
                borrowernumber     => $borrowernumber,
                cardnumber         => $borrower{cardnumber}
              };
            $invalid++;
            next;
        }

        if ($borrowernumber) {

            # borrower exists
            unless ($overwrite_cardnumber) {
                $alreadyindb++;
                $template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber );
                next LINE;
            }
            $borrower{'borrowernumber'} = $borrowernumber;
            for my $col ( keys %borrower ) {

                # use values from extant patron unless our csv file includes this column or we provided a default.
                # FIXME : You cannot update a field with a  perl-evaluated false value using the defaults.

                # The password is always encrypted, skip it!
                next if $col eq 'password';

                unless ( exists( $csvkeycol{$col} ) || $defaults{$col} ) {
                    $borrower{$col} = $member->{$col} if ( $member->{$col} );
                }
            }
            unless ( ModMember(%borrower) ) {
                $invalid++;

                # untill we have better error trapping, we have no way of knowing why ModMember errored out...
                push @errors, { unknown_error => 1 };
                $template->param( 'lastinvalid' => $borrower{'surname'} . ' / ' . $borrowernumber );
                next LINE;
            }
            if ( $borrower{debarred} ) {

                # Check to see if this debarment already exists
                my $debarrments = GetDebarments(
                    {
                        borrowernumber => $borrowernumber,
                        expiration     => $borrower{debarred},
                        comment        => $borrower{debarredcomment}
                    }
                );

                # If it doesn't, then add it!
                unless (@$debarrments) {
                    AddDebarment(
                        {
                            borrowernumber => $borrowernumber,
                            expiration     => $borrower{debarred},
                            comment        => $borrower{debarredcomment}
                        }
                    );
                }
            }
            if ($extended) {
                if ($ext_preserve) {
                    my $old_attributes = GetBorrowerAttributes($borrowernumber);
                    $patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes )
                      ;    #TODO: expose repeatable options in template
                }
                push @errors, { unknown_error => 1 }
                  unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
            }
            $overwritten++;
            $template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber );
        }
        else {
            # FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
            # At least this is closer to AddMember than in members/memberentry.pl
            if ( !$borrower{'cardnumber'} ) {
                $borrower{'cardnumber'} = fixup_cardnumber(undef);
            }
            if ( $borrowernumber = AddMember(%borrower) ) {

    my $feedback    = $return->{feedback};
    my $errors      = $return->{errors};
    my $imported    = $return->{imported};
    my $overwritten = $return->{overwritten};
    my $alreadyindb = $return->{already_in_db};
    my $invalid     = $return->{invalid};
                        }
                    );
                }

    my $uploadinfo = $input->uploadInfo($uploadborrowers);
    foreach ( keys %$uploadinfo ) {
        push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
    }

    push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };
                    C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
                        {
                            borrowernumber => $borrowernumber,
                            categorycode   => $borrower{categorycode}
                        }
                    );
                }

                $imported++;
                $template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber );
            }
            else {
                $invalid++;
                push @errors, { unknown_error => 1 };
                $template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember' );
            }
        }
    }
    (@errors)   and $template->param( ERRORS   => \@errors );
    (@feedback) and $template->param( FEEDBACK => \@feedback );
    $template->param(
        uploadborrowers => 1,
        errors          => $errors,
        feedback        => $feedback,
        imported        => $imported,
        overwritten     => $overwritten,
        alreadyindb     => $alreadyindb,
        invalid         => $invalid,
        total           => $imported + $alreadyindb + $invalid + $overwritten,
    );

}
- 

Return to bug 12598

Line 0 Link Here

(-)a/Koha/Patrons/Import.pm (+370 lines)
		1	package Koha::Patrons::Import;
		2
		3	# This file is part of Koha.
		4	#
		5	# Koha is free software; you can redistribute it and/or modify it under the
		6	# terms of the GNU General Public License as published by the Free Software
		7	# Foundation; either version 3 of the License, or (at your option) any later
		8	# version.
		9	#
		10	# Koha is distributed in the hope that it will be useful, but WITHOUT ANY
		11	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
		12	# A PARTICULAR PURPOSE. See the GNU General Public License for more details.
		13	#
		14	# You should have received a copy of the GNU General Public License along
		15	# with Koha; if not, write to the Free Software Foundation, Inc.,
		16	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		17
		18	use Modern::Perl;
		19
		20	use Carp;
		21	use Text::CSV;
		22
		23	use C4::Members;
		24	use C4::Branch;
		25
		26	sub import_patrons {
		27	my ($params) = @_;
		28
		29	my $handle = $params->{file};
		30	my $matchpoint = $params->{matchpoint};
		31	my $defaults = $params->{defaults};
		32	my $ext_preserve = $params->{preserve_extended_attributes};
		33	my $overwrite_cardnumber = $params->{overwrite_cardnumber};
		34
		35	carp("No file handle passed in!") && return unless $handle;
		36
		37	my $extended = C4::Context->preference('ExtendedPatronAttributes');
		38	my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');
		39
		40	my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
		41	push( @columnkeys, 'patron_attributes' ) if $extended;
		42
		43	our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode
		44
		45	my @feedback;
		46	my @errors;
		47
		48	my $imported = 0;
		49	my $alreadyindb = 0;
		50	my $overwritten = 0;
		51	my $invalid = 0;
		52	my $matchpoint_attr_type;
		53
		54	# use header line to construct key to column map
		55	my $borrowerline = <$handle>;
		56	my $status = $csv->parse($borrowerline);
		57	($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline };
		58	my @csvcolumns = $csv->fields();
		59	my %csvkeycol;
		60	my $col = 0;
		61	foreach my $keycol (@csvcolumns) {
		62
		63	# columnkeys don't contain whitespace, but some stupid tools add it
		64	$keycol =~ s/ +//g;
65	$csvkeycol{$keycol} = $col++;
66	}
67
68	#warn($borrowerline);
69	if ($extended) {
70	$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
71	}
72
73	push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
74	my $today_iso = C4::Dates->new()->output('iso');
75	my @criticals = qw(surname branchcode categorycode); # there probably should be others
76	my @bad_dates; # I've had a few.
77	my $date_re = C4::Dates->new->regexp('syspref');
78	my $iso_re = C4::Dates->new->regexp('iso');
79	LINE: while ( my $borrowerline = <$handle> ) {
80	my %borrower;
81	my @missing_criticals;
82	my $patron_attributes;
83	my $status = $csv->parse($borrowerline);
84	my @columns = $csv->fields();
85	if ( !$status ) {
86	push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
87	}
88	elsif ( @columns == @columnkeys ) {
89	@borrower{@columnkeys} = @columns;
90
91	# MJR: try to fill blanks gracefully by using default values
92	foreach my $key (@columnkeys) {
93	if ( $borrower{$key} !~ /\S/ ) {
94	$borrower{$key} = $defaults->{$key};
95	}
96	}
97	}
98	else {
99	# MJR: try to recover gracefully by using default values
100	foreach my $key (@columnkeys) {
101	if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
102	$borrower{$key} = $columns[ $csvkeycol{$key} ];
103	}
104	elsif ( $defaults->{$key} ) {
105	$borrower{$key} = $defaults->{$key};
106	}
107	elsif ( scalar grep { $key eq $_ } @criticals ) {
108
109	# a critical field is undefined
110	push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
111	}
112	else {
113	$borrower{$key} = '';
114	}
115	}
116	}
117
118	#warn join(':',%borrower);
119	if ( $borrower{categorycode} ) {
120	push @missing_criticals,
121	{
122	key => 'categorycode',
123	line => $.,
124	lineraw => $borrowerline,
125	value => $borrower{categorycode},
126	category_map => 1
127	}
128	unless GetBorrowercategory( $borrower{categorycode} );
129	}
130	else {
131	push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
132	}
133	if ( $borrower{branchcode} ) {
134	push @missing_criticals,
135	{
136	key => 'branchcode',
137	line => $.,
138	lineraw => $borrowerline,
139	value => $borrower{branchcode},
140	branch_map => 1
141	}
142	unless GetBranchName( $borrower{branchcode} );
143	}
144	else {
145	push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
146	}
147	if (@missing_criticals) {
148	foreach (@missing_criticals) {
149	$_->{borrowernumber} = $borrower{borrowernumber} \|\| 'UNDEF';
150	$_->{surname} = $borrower{surname} \|\| 'UNDEF';
151	}
152	$invalid++;
153	( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };
154
155	# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance.
156	next LINE;
157	}
158	if ($extended) {
159	my $attr_str = $borrower{patron_attributes};
160	$attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes
161	$attr_str =~ s/\xe2\x80\x9d/"/g;
162	push @feedback, { feedback => 1, name => 'attribute string', value => $attr_str };
163	delete $borrower{patron_attributes}
164	; # not really a field in borrowers, so we don't want to pass it to ModMember.
165	$patron_attributes = extended_attributes_code_value_arrayref($attr_str);
166	}
167
168	# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
169	foreach (qw(dateofbirth dateenrolled dateexpiry)) {
170	my $tempdate = $borrower{$_} or next;
171	if ( $tempdate =~ /$date_re/ ) {
172	$borrower{$_} = format_date_in_iso($tempdate);
173	}
174	elsif ( $tempdate =~ /$iso_re/ ) {
175	$borrower{$_} = $tempdate;
176	}
177	else {
178	$borrower{$_} = '';
179	push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
180	}
181	}
182	$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
183	$borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
184	unless $borrower{dateexpiry};
185	my $borrowernumber;
186	my $member;
187	if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
188	$member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
189	if ($member) {
190	$borrowernumber = $member->{'borrowernumber'};
191	}
192	}
193	elsif ($extended) {
194	if ( defined($matchpoint_attr_type) ) {
195	foreach my $attr (@$patron_attributes) {
196	if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
197	my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
198	$borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
199	last;
200	}
201	}
202	}
203	}
204
205	if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
206	push @errors,
207	{
208	invalid_cardnumber => 1,
209	borrowernumber => $borrowernumber,
210	cardnumber => $borrower{cardnumber}
211	};
212	$invalid++;
213	next;
214	}
215
216	if ($borrowernumber) {
217
218	# borrower exists
219	unless ($overwrite_cardnumber) {
220	$alreadyindb++;
221	push(
222	@feedback,
223	{
224	already_in_db => 1,
225	value => $borrower{'surname'} . ' / ' . $borrowernumber
226	}
227	);
228	next LINE;
229	}
230	$borrower{'borrowernumber'} = $borrowernumber;
231	for my $col ( keys %borrower ) {
232
233	# use values from extant patron unless our csv file includes this column or we provided a default.
234	# FIXME : You cannot update a field with a perl-evaluated false value using the defaults.
235
236	# The password is always encrypted, skip it!
237	next if $col eq 'password';
238
239	unless ( exists( $csvkeycol{$col} ) \|\| $defaults->{$col} ) {
240	$borrower{$col} = $member->{$col} if ( $member->{$col} );
241	}
242	}
243	unless ( ModMember(%borrower) ) {
244	$invalid++;
245
246	push(
247	@errors,
248	{
249	name => 'lastinvalid',
250	value => $borrower{'surname'} . ' / ' . $borrowernumber
251	}
252	);
253	next LINE;
254	}
255	if ( $borrower{debarred} ) {
256
257	# Check to see if this debarment already exists
258	my $debarrments = GetDebarments(
259	{
260	borrowernumber => $borrowernumber,
261	expiration => $borrower{debarred},
262	comment => $borrower{debarredcomment}
263	}
264	);
265
266	# If it doesn't, then add it!
267	unless (@$debarrments) {
268	AddDebarment(
269	{
270	borrowernumber => $borrowernumber,
271	expiration => $borrower{debarred},
272	comment => $borrower{debarredcomment}
273	}
274	);
275	}
276	}
277	if ($extended) {
278	if ($ext_preserve) {
279	my $old_attributes = GetBorrowerAttributes($borrowernumber);
280	$patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes );
281	}
282	push @errors, { unknown_error => 1 }
283	unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
284	}
285	$overwritten++;
286	push(
287	@feedback,
288	{
289	feedback => 1,
290	name => 'lastoverwritten',
291	value => $borrower{'surname'} . ' / ' . $borrowernumber
292	}
293	);
294	}
295	else {
296	# FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
297	# At least this is closer to AddMember than in members/memberentry.pl
298	if ( !$borrower{'cardnumber'} ) {
299	$borrower{'cardnumber'} = fixup_cardnumber(undef);
300	}
301	if ( $borrowernumber = AddMember(%borrower) ) {
302
303	if ( $borrower{debarred} ) {
304	AddDebarment(
305	{
306	borrowernumber => $borrowernumber,
307	expiration => $borrower{debarred},
308	comment => $borrower{debarredcomment}
309	}
310	);
311	}
312
313	if ($extended) {
314	SetBorrowerAttributes( $borrowernumber, $patron_attributes );
315	}
316
317	if ($set_messaging_prefs) {
318	C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
319	{
320	borrowernumber => $borrowernumber,
321	categorycode => $borrower{categorycode}
322	}
323	);
324	}
325
326	$imported++;
327	push(
328	@feedback,
329	{
330	feedback => 1,
331	name => 'lastimported',
332	value => $borrower{'surname'} . ' / ' . $borrowernumber
333	}
334	);
335	}
336	else {
337	$invalid++;
338	push @errors, { unknown_error => 1 };
339	push(
340	@errors,
341	{
342	name => 'lastinvalid',
343	value => $borrower{'surname'} . ' / AddMember',
344	}
345	);
346	}
347	}
348	}
349
350	return {
351	feedback => \@feedback,
352	errors => \@errors,
353	imported => $imported,
354	overwritten => $overwritten,
355	already_in_db => $alreadyindb,
356	invalid => $invalid,
357	};
358	}
359
360	END { } # module clean-up code here (global destructor)
361
362	1;
363
364	__END__
365
366	=head1 AUTHOR
367
368	Koha Team
369
370	=cut

Lines 33-220 Link Here

(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/import_borrowers.tt (-172 / +233 lines)
33	<div class="yui-u first">	33	<div class="yui-u first">
34	<h1>Import patrons</h1>	34	<h1>Import patrons</h1>
35	[% IF ( uploadborrowers ) %]	35	[% IF ( uploadborrowers ) %]
36	<h5>Import results :</h5>	36	<h5>Import results :</h5>
37	<ul>
38	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
39	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
40	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
41	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
42	<li>[% total %] records parsed</li>
43	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
44	</ul>
45	[% IF ( FEEDBACK ) %]
46	<br /><br />
47	<div>
48	<h5>Feedback:</h5>
49	<ul class="feedback">
50	[% FOREACH FEEDBAC IN FEEDBACK %]
51	<li>
52	[% IF ( FEEDBAC.filename ) %]Parsing upload file <span class="filename">[% FEEDBAC.filename %]</span>
53	[% ELSIF ( FEEDBAC.backend ) %]Upload parsed using [% FEEDBAC.backend %]
54	[% ELSIF ( FEEDBAC.headerrow ) %]These fields found: [% FEEDBAC.value %]
55	[% ELSE %][% FEEDBAC.name %] : [% FEEDBAC.value %]
56	[% END %]
57	</li>
58	[% END %]
59	</ul>
60	</div>
61	[% END %]
62	[% IF ( ERRORS ) %]
63	<br /><br />
64	<div>
65	<h5>Error analysis:</h5>
66	<ul>	37	<ul>
67	[% FOREACH ERROR IN ERRORS %]	38	<li>[% imported %] imported records [% IF ( lastimported ) %](last was [% lastimported %])[% END %]</li>
68	[% IF ( ERROR.badheader ) %]<li>Header row could not be parsed</li>[% END %]	39	<li>[% overwritten %] overwritten [% IF ( lastoverwritten ) %](last was [% lastoverwritten %])[% END %]</li>
69	[% FOREACH missing_critical IN ERROR.missing_criticals %]	40	<li>[% alreadyindb %] not imported because already in borrowers table and overwrite disabled [% IF ( lastalreadyindb ) %](last was [% lastalreadyindb %])[% END %]</li>
70	<li class="line_error">	41	<li>[% invalid %] not imported because they are not in the expected format [% IF ( lastinvalid ) %](last was [% lastinvalid %])[% END %]</li>
71	Line <span class="linenumber">[% missing_critical.line %]</span>	42	<li>[% total %] records parsed</li>
72	[% IF ( missing_critical.badparse ) %]	43	<li><a href="/cgi-bin/koha/tools/tools-home.pl">Back to Tools</a></li>
73	could not be parsed!	44	</ul>
74	[% ELSIF ( missing_critical.bad_date ) %]	45
75	has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]"	46	[% IF ( feedback ) %]
76	[% ELSE %]	47	<br /><br />
77	Critical field "[% missing_critical.key %]"	48
78	[% IF ( missing_critical.branch_map ) %]has unrecognized value "[% missing_critical.value %]"	49	<div>
79	[% ELSIF ( missing_critical.category_map ) %]has unrecognized value "[% missing_critical.value %]"	50	<h5>Feedback:</h5>
80	[% ELSE %]missing	51	<ul class="feedback">
		52	[% FOREACH f IN feedback %]
		53	<li>
		54	[% IF ( f.filename ) %]
		55	Parsing upload file <span class="filename">[% f.filename %]</span>
		56	[% ELSIF ( f.backend ) %]
		57	Upload parsed using [% f.backend %]
		58	[% ELSIF ( f.headerrow ) %]
		59	These fields found: [% f.value %]
		60	[% ELSIF ( f.already_in_db ) %]
		61	Patron already in database: [% f.value %]
		62	[% ELSE %]
		63	[% f.name %] : [% f.value %]
		64	[% END %]
		65	</li>
		66	[% END %]
		67	</ul>
		68	</div>
		69	[% END %]
		70
		71	[% IF ( errors ) %]
		72	<br /><br />
		73
		74	<div>
		75	<h5>Error analysis:</h5>
		76	<ul>
		77	[% FOREACH e IN errors %]
		78	[% IF ( e.badheader ) %]<li>Header row could not be parsed</li>[% END %]
		79
		80	[% FOREACH missing_critical IN e.missing_criticals %]
		81	<li class="line_error">
		82	Line <span class="linenumber">[% missing_critical.line %]</span>
		83
		84	[% IF ( missing_critical.badparse ) %]
		85	could not be parsed!
		86	[% ELSIF ( missing_critical.bad_date ) %]
		87	has "[% missing_critical.key %]" in unrecognized format: "[% missing_critical.value %]"
		88	[% ELSE %]
		89	Critical field "[% missing_critical.key %]"
		90
		91	[% IF ( missing_critical.branch_map ) %]
		92	has unrecognized value "[% missing_critical.value %]"
		93	[% ELSIF ( missing_critical.category_map ) %]
		94	has unrecognized value "[% missing_critical.value %]"
		95	[% ELSE %]
		96	missing
		97	[% END %]
		98
		99	(borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).
		100	[% END %]
		101
		102	<br/>
		103	<code>[% missing_critical.lineraw %]</code>
		104	</li>
		105	[% END %]
		106
		107	[% IF e.invalid_cardnumber %]
		108	<li class="line_error">
		109	Cardnumber [% e.cardnumber %] is not a valid cardnumber
		110	[% IF e.borrowernumber %] (for patron with borrowernumber [% e.borrowernumber %])[% END %]
		111	</li>
		112	[% END %]
81	[% END %]	113	[% END %]
82	(borrowernumber: [% missing_critical.borrowernumber %]; surname: [% missing_critical.surname %]).	114	</ul>
83	[% END %]	115	</div>
84	<br /><code>[% missing_critical.lineraw %]</code>
85	</li>
86	[% END %]
87	[% IF ERROR.invalid_cardnumber %]
88	<li class="line_error">
89	Cardnumber [% ERROR.cardnumber %] is not a valid cardnumber
90	[% IF ERROR.borrowernumber %] (for patron with borrowernumber [% ERROR.borrowernumber %])[% END %]
91	</li>
92	[% END %]
93	[% END %]	116	[% END %]
94	</ul>
95	</div>
96	[% END %]
97	[% ELSE %]	117	[% ELSE %]
98	<ul>	118	<ul>
99	<li>Select a file to import into the borrowers table</li>	119	<li>Select a file to import into the borrowers table</li>
100	<li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>	120	<li>If a cardnumber exists in the table, you can choose whether to ignore the new one or overwrite the old one.</li>
101	</ul>	121	</ul>
102	<form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">	122
103	<fieldset class="rows">	123	<form method="post" action="[% SCRIPT_NAME %]" enctype="multipart/form-data">
104	<legend>Import into the borrowers table</legend>	124	<fieldset class="rows">
105	<ol>	125	<legend>Import into the borrowers table</legend>
106	<li>	126
107	<label for="uploadborrowers">Select the file to import: </label>	127	<ol>
108	<input type="file" id="uploadborrowers" name="uploadborrowers" />	128	<li>
109	</li>	129	<label for="uploadborrowers">Select the file to import: </label>
110	</ol></fieldset>	130	<input type="file" id="uploadborrowers" name="uploadborrowers" />
111	<fieldset class="rows">	131	</li>
112	<legend>Field to use for record matching</legend>	132	</ol>
113	<ol>	133	</fieldset>
114	<li class="radio">	134
115	<select name="matchpoint" id="matchpoint">	135	<fieldset class="rows">
116	<option value="cardnumber">Cardnumber</option>	136	<legend>Field to use for record matching</legend>
117	<option value="userid">Username</option>	137	<ol>
118	[% FOREACH matchpoint IN matchpoints %]	138	<li class="radio">
119	<option value="[% matchpoint.code %]">[% matchpoint.description %]</option>	139	<select name="matchpoint" id="matchpoint">
		140	<option value="cardnumber">Cardnumber</option>
		141	[% FOREACH matchpoint IN matchpoints %]
		142	<option value="[% matchpoint.code %]">[% matchpoint.description %]</option>
		143	[% END %]
		144	</select>
		145	</li>
		146	</ol>
		147	</fieldset>
		148
		149	<fieldset class="rows">
		150	<legend>Default values</legend>
		151
		152	<ol>
		153	[% FOREACH borrower_field IN borrower_fields %]
		154
		155	[% SWITCH borrower_field.field %]
		156	[% CASE 'branchcode' %]
		157	<li>
		158	<label class="description" for="branchcode">[% borrower_field.description %]: </label>
		159	<select id="branchcode" name="branchcode">
		160	<option value="" selected="selected"></option>
		161	[% FOREACH branch IN branches %]
		162	<option value="[% branch.branchcode %]">[% branch.branchname %]</option>
		163	[% END %]
		164	</select>
		165	<span class="field_hint">[% borrower_field.field %]</span>
		166	</li>
		167	[% CASE 'categorycode' %]
		168	<li>
		169	<label class="description" for="categorycode">[% borrower_field.description %]: </label>
		170	<select id="categorycode" name="categorycode">
		171	<option value="" selected="selected"></option>
		172	[% FOREACH category IN categories %]
		173	<option value="[% category.categorycode %]">[% category.description %]</option>
		174	[% END %]
		175	</select>
		176	<span class="field_hint">[% borrower_field.field %]</span>
		177	</li>
		178	[% CASE %]
		179	<li>
		180	<label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
		181	<input id="[% borrower_field.field %]" name="[% borrower_field.field %]" />
		182	<span class="field_hint">[% borrower_field.field %]</span>
		183	</li>
120	[% END %]	184	[% END %]
121	</select>	185
122	</li>	186	[% END %]
123	</ol>	187
124	</fieldset>	188	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
125	<fieldset class="rows">	189	<li>
126	<legend>Default values</legend>	190	<label class="description" for="patron_attributes">Patron attributes: </label>
127	<ol>	191	<input id="patron_attributes" name="patron_attributes" />
128	[% FOREACH borrower_field IN borrower_fields %]	192	<span class="field_hint">patron_attributes</span>
129	[% SWITCH borrower_field.field %]	193	</li>
130	[% CASE 'branchcode' %]	194	[% END %]
131	<li>	195
132	<label class="description" for="branchcode">[% borrower_field.description %]: </label>	196	</ol>
133	<select id="branchcode" name="branchcode">	197	</fieldset>
134	<option value="" selected="selected"></option>	198
135	[% FOREACH branch IN branches %]	199	<fieldset class="rows">
136	<option value="[% branch.branchcode %]">	200	<legend>If matching record is already in the borrowers table:</legend>
137	[% branch.branchname %]</option>	201
138	[% END %]	202	<ol>
139	</select><span class="field_hint">[% borrower_field.field %]</span>	203	<li class="radio">
140	</li>	204	<input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>
141	[% CASE 'categorycode' %]	205	</li>
142	<li>	206
143	<label class="description" for="categorycode">[% borrower_field.description %]: </label>	207	<li class="radio">
144	<select id="categorycode" name="categorycode">	208	<input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>
145	<option value="" selected="selected"></option>	209	</li>
146	[% FOREACH category IN categories %]	210	</ol>
147	<option value="[% category.categorycode %]">	211	</fieldset>
148	[% category.description %]</option>	212
		213	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
		214	<fieldset class="rows">
		215	<legend>Patron attributes</legend>
		216
		217	<ol>
		218	<li class="radio">
		219	<input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
		220	</li>
		221
		222	<li class="radio">
		223	<input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>
		224	</li>
		225	</ol>
		226	</fieldset>
149	[% END %]	227	[% END %]
150	</select><span class="field_hint">[% borrower_field.field %]</span>	228
151	</li>	229	<fieldset class="action"><input type="submit" value="Import" /></fieldset>
152	[% CASE %]	230	</form>
153	<li>
154	<label class="description" for="[% borrower_field.field %]">[% borrower_field.description %]: </label>
155	<input id="[% borrower_field.field %]" name="[% borrower_field.field %]" /><span class="field_hint">[% borrower_field.field %]</span>
156	</li>
157	[% END %]
158	[% END %]
159	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]
160	<li>
161	<label class="description" for="patron_attributes">Patron attributes: </label>
162	<input id="patron_attributes" name="patron_attributes" />
163	<span class="field_hint">patron_attributes</span>
164	</li>
165	[% END %]	231	[% END %]
166	</ol></fieldset>	232
167	<fieldset class="rows">	233	</div>
168	<legend>If matching record is already in the borrowers table:</legend>	234
169	<ol><li class="radio">	235	<div class="yui-u">
170	<input type="radio" id="overwrite_cardnumberno" name="overwrite_cardnumber" value="0" checked="checked" /><label for="overwrite_cardnumberno">Ignore this one, keep the existing one</label>	236	<h2>Notes:</h2>
		237	<ul>
		238	<li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>
		239
		240	<li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b> Values are comma-separated.</li>
		241
		242	<li>
		243	OR choose which fields you want to supply from the following list:
		244	<ul>
		245	<li>
		246	[% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]
		247	</li>
		248	</ul>
171	</li>	249	</li>
172	<li class="radio">	250
173	<input type="radio" id="overwrite_cardnumberyes" name="overwrite_cardnumber" value="1" /><label for="overwrite_cardnumberyes">Overwrite the existing one with this</label>	251	[% IF ( ExtendedPatronAttributes ) %]
		252	<li>
		253	If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>"STARTDATE:January 1, 2010","TRACK:Day"</b>. The second syntax would be required if the data might have a comma in it, like a date string.
		254	</li>
		255	[% END %]
		256
		257	<li>
		258	The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.
174	</li>	259	</li>
175	</ol>	260
176	</fieldset>	261	<li>
177	[% IF ( Koha.Preference('ExtendedPatronAttributes') == 1 ) %]	262	'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).
178	<fieldset class="rows">
179	<legend>Patron attributes</legend>
180	<ol><li class="radio">
181	<input type="radio" id="ext_preserve_0" name="ext_preserve" value="0" checked="checked" /><label for="ext_preserve_0">Replace all patron attributes</label>
182	</li>	263	</li>
183	<li class="radio">	264
184	<input type="radio" id="ext_preserve_1" name="ext_preserve" value="1" /><label for="ext_preserve_1">Replace only included patron attributes</label>	265	<li>
		266	Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'. Alternatively,
		267	you can supply dates in ISO format (e.g., '2010-10-28').
185	</li>	268	</li>
186	</ol>	269	</ul>
187	</fieldset>	270
188	[% END %]
189	<fieldset class="action"><input type="submit" value="Import" /></fieldset>
190	</form>
191	[% END %]
192	</div>	271	</div>
193	<div class="yui-u">	272	</div>
194	<h2>Notes:</h2>	273	</div>
195	<ul>	274	</div>
196	<li>The first line in the file must be a header row defining which columns you are supplying in the import file.</li>	275
197	<li><b>Download a starter CSV file with all the columns <a href="?sample=1">here</a>.</b> Values are comma-separated.</li>	276	<div class="yui-b noprint">
198	<li>OR choose which fields you want to supply from the following list:<ul><li>	277	[% INCLUDE 'tools-menu.inc' %]
199	[% FOREACH columnkey IN borrower_fields %]'[% columnkey.field %]', [% END %]	278	</div>
200	</li></ul></li>	279	</div>
201	[% IF ( ExtendedPatronAttributes ) %]	280
202	<li>If loading patron attributes, the 'patron_attributes' field should contain a comma-separated list of attribute types and values. The attribute type code and a colon should precede each value. For example: <b>INSTID:12345,LANG:fr</b> or <b>STARTDATE:January 1 2010,TRACK:Day</b>. If an input record has more than one attribute, the fields should either be entered as an unquoted string (previous examples), or with each field wrapped in separate double quotes and delimited by a comma: <b>"STARTDATE:January 1, 2010","TRACK:Day"</b>. The second syntax would be required if the data might have a comma in it, like a date string.
203	</li>
204	[% END %]
205	<li>The fields 'branchcode' and 'categorycode' are <b>required</b> and <b>must match</b> valid entries in your database.</li>
206	<li>'password' should be stored in plaintext, and will be converted to a Bcrypt hash (if your passwords are already encrypted, talk to your system administrator about options).</li>
207	<li>Date formats should match your system preference, and <b>must</b> be zero-padded, e.g. '01/02/2008'. Alternatively,
208	you can supply dates in ISO format (e.g., '2010-10-28').
209	</li>
210	</ul>
211
212	</div>
213	</div>
214	</div>
215	</div>
216	<div class="yui-b noprint">
217	[% INCLUDE 'tools-menu.inc' %]
218	</div>
219	</div>
220	[% INCLUDE 'intranet-bottom.inc' %]	281	[% INCLUDE 'intranet-bottom.inc' %]

Lines 34-54 Link Here

(-)a/tools/import_borrowers.pl (-307 / +40 lines)
34	# dates should be in the format you have set up Koha to expect	34	# dates should be in the format you have set up Koha to expect
35	# branchcode and categorycode need to be valid	35	# branchcode and categorycode need to be valid
36		36
37	use strict;	37	use Modern::Perl;
38	use warnings;
39		38
40	use C4::Auth;	39	use C4::Auth;
41	use C4::Output;	40	use C4::Output;
42	use C4::Dates qw(format_date_in_iso);
43	use C4::Context;
44	use C4::Branch qw/GetBranchesLoop GetBranchName/;
45	use C4::Members;
46	use C4::Members::Attributes qw(:all);
47	use C4::Members::AttributeTypes;
48	use C4::Members::Messaging;
49	use C4::Reports::Guided;
50	use C4::Templates;	41	use C4::Templates;
51	use Koha::Borrower::Debarments;	42
		43	use C4::Branch;
		44	use C4::Members;
		45
		46	use Koha::Patrons::Import qw(import_patrons);
52		47
53	use Text::CSV;	48	use Text::CSV;
54		49
Lines 61-76 use CGI qw ( -utf8 ); Link Here
61	# use encoding 'utf8'; # don't do this	56	# use encoding 'utf8'; # don't do this
62		57
63	my ( @errors, @feedback );	58	my ( @errors, @feedback );
64	my $extended = C4::Context->preference('ExtendedPatronAttributes');	59	my $extended = C4::Context->preference('ExtendedPatronAttributes');
65	my $set_messaging_prefs = C4::Context->preference('EnhancedMessagingPreferences');	60
66	my @columnkeys = C4::Members::columns();	61	my @columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } C4::Members::columns();
67	@columnkeys = map { $_ ne 'borrowernumber' ? $_ : () } @columnkeys;	62	push( @columnkeys, 'patron_attributes' ) if $extended;
68	if ($extended) {
69	push @columnkeys, 'patron_attributes';
70	}
71		63
72	my $input = CGI->new();	64	my $input = CGI->new();
73	our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode
74		65
75	#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX	66	#push @feedback, {feedback=>1, name=>'backend', value=>$csv->backend, backend=>$csv->backend}; #XXX
76		67
Lines 97-102 $columns = [ grep { $_->{field} ne 'borrowernumber' ? $_ : () } @$columns ]; Link Here
97	$template->param( borrower_fields => $columns );	88	$template->param( borrower_fields => $columns );
98		89
99	if ( $input->param('sample') ) {	90	if ( $input->param('sample') ) {
		91	our $csv = Text::CSV->new( { binary => 1 } ); # binary needed for non-ASCII Unicode
100	print $input->header(	92	print $input->header(
101	-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?	93	-type => 'application/vnd.sun.xml.calc', # 'application/vnd.ms-excel' ?
102	-attachment => 'patron_import.csv',	94	-attachment => 'patron_import.csv',
Lines 105-414 if ( $input->param('sample') ) { Link Here
105	print $csv->string, "\n";	97	print $csv->string, "\n";
106	exit 1;	98	exit 1;
107	}	99	}
		100
108	my $uploadborrowers = $input->param('uploadborrowers');	101	my $uploadborrowers = $input->param('uploadborrowers');
109	my $matchpoint = $input->param('matchpoint');	102	my $matchpoint = $input->param('matchpoint');
110	if ($matchpoint) {	103	if ($matchpoint) {
111	$matchpoint =~ s/^patron_attribute_//;	104	$matchpoint =~ s/^patron_attribute_//;
112	}	105	}
113	my $overwrite_cardnumber = $input->param('overwrite_cardnumber');
114		106
115	$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );	107	$template->param( SCRIPT_NAME => $ENV{'SCRIPT_NAME'} );
116		108
117	if ( $uploadborrowers && length($uploadborrowers) > 0 ) {	109	if ( $uploadborrowers && length($uploadborrowers) > 0 ) {
118	push @feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };	110	my $handle = $input->upload('uploadborrowers');
119	my $handle = $input->upload('uploadborrowers');
120	my $uploadinfo = $input->uploadInfo($uploadborrowers);
121	foreach ( keys %$uploadinfo ) {
122	push @feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
123	}
124	my $imported = 0;
125	my $alreadyindb = 0;
126	my $overwritten = 0;
127	my $invalid = 0;
128	my $matchpoint_attr_type;
129	my %defaults = $input->Vars;	111	my %defaults = $input->Vars;
130		112
131	# use header line to construct key to column map	113	my $return = Koha::Patrons::Import::import_patrons(
132	my $borrowerline = <$handle>;	114	{
133	my $status = $csv->parse($borrowerline);	115	file => $handle,
134	($status) or push @errors, { badheader => 1, line => $., lineraw => $borrowerline };	116	defaults => \%defaults,
135	my @csvcolumns = $csv->fields();	117	matchpoint => $matchpoint,
136	my %csvkeycol;	118	overwrite_cardnumber => $input->param('overwrite_cardnumber'),
137	my $col = 0;	119	preserve_extended_attributes => $input->param('ext_preserve') \|\| 0,
138	foreach my $keycol (@csvcolumns) {
139
140	# columnkeys don't contain whitespace, but some stupid tools add it
141	$keycol =~ s/ +//g;
142	$csvkeycol{$keycol} = $col++;
143	}
144
145	#warn($borrowerline);
146	my $ext_preserve = $input->param('ext_preserve') \|\| 0;
147	if ($extended) {
148	$matchpoint_attr_type = C4::Members::AttributeTypes->fetch($matchpoint);
149	}
150
151	push @feedback, { feedback => 1, name => 'headerrow', value => join( ', ', @csvcolumns ) };
152	my $today_iso = C4::Dates->new()->output('iso');
153	my @criticals = qw(surname branchcode categorycode); # there probably should be others
154	my @bad_dates; # I've had a few.
155	my $date_re = C4::Dates->new->regexp('syspref');
156	my $iso_re = C4::Dates->new->regexp('iso');
157	LINE: while ( my $borrowerline = <$handle> ) {
158	my %borrower;
159	my @missing_criticals;
160	my $patron_attributes;
161	my $status = $csv->parse($borrowerline);
162	my @columns = $csv->fields();
163	if ( !$status ) {
164	push @missing_criticals, { badparse => 1, line => $., lineraw => $borrowerline };
165	}	120	}
166	elsif ( @columns == @columnkeys ) {	121	);
167	@borrower{@columnkeys} = @columns;
168
169	# MJR: try to fill blanks gracefully by using default values
170	foreach my $key (@columnkeys) {
171	if ( $borrower{$key} !~ /\S/ ) {
172	$borrower{$key} = $defaults{$key};
173	}
174	}
175	}
176	else {
177	# MJR: try to recover gracefully by using default values
178	foreach my $key (@columnkeys) {
179	if ( defined( $csvkeycol{$key} ) and $columns[ $csvkeycol{$key} ] =~ /\S/ ) {
180	$borrower{$key} = $columns[ $csvkeycol{$key} ];
181	}
182	elsif ( $defaults{$key} ) {
183	$borrower{$key} = $defaults{$key};
184	}
185	elsif ( scalar grep { $key eq $_ } @criticals ) {
186
187	# a critical field is undefined
188	push @missing_criticals, { key => $key, line => $., lineraw => $borrowerline };
189	}
190	else {
191	$borrower{$key} = '';
192	}
193	}
194	}
195
196	#warn join(':',%borrower);
197	if ( $borrower{categorycode} ) {
198	push @missing_criticals,
199	{
200	key => 'categorycode',
201	line => $.,
202	lineraw => $borrowerline,
203	value => $borrower{categorycode},
204	category_map => 1
205	}
206	unless GetBorrowercategory( $borrower{categorycode} );
207	}
208	else {
209	push @missing_criticals, { key => 'categorycode', line => $., lineraw => $borrowerline };
210	}
211	if ( $borrower{branchcode} ) {
212	push @missing_criticals,
213	{
214	key => 'branchcode',
215	line => $.,
216	lineraw => $borrowerline,
217	value => $borrower{branchcode},
218	branch_map => 1
219	}
220	unless GetBranchName( $borrower{branchcode} );
221	}
222	else {
223	push @missing_criticals, { key => 'branchcode', line => $., lineraw => $borrowerline };
224	}
225	if (@missing_criticals) {
226	foreach (@missing_criticals) {
227	$_->{borrowernumber} = $borrower{borrowernumber} \|\| 'UNDEF';
228	$_->{surname} = $borrower{surname} \|\| 'UNDEF';
229	}
230	$invalid++;
231	( 25 > scalar @errors ) and push @errors, { missing_criticals => \@missing_criticals };
232
233	# The first 25 errors are enough. Keeping track of 30,000+ would destroy performance.
234	next LINE;
235	}
236	if ($extended) {
237	my $attr_str = $borrower{patron_attributes};
238	$attr_str =~ s/\xe2\x80\x9c/"/g; # fixup double quotes in case we are passed smart quotes
239	$attr_str =~ s/\xe2\x80\x9d/"/g;
240	push @feedback,
241	{ feedback => 1, name => 'attribute string', value => $attr_str, filename => $uploadborrowers };
242	delete $borrower{patron_attributes}
243	; # not really a field in borrowers, so we don't want to pass it to ModMember.
244	$patron_attributes = extended_attributes_code_value_arrayref($attr_str);
245	}
246
247	# Popular spreadsheet applications make it difficult to force date outputs to be zero-padded, but we require it.
248	foreach (qw(dateofbirth dateenrolled dateexpiry)) {
249	my $tempdate = $borrower{$_} or next;
250	if ( $tempdate =~ /$date_re/ ) {
251	$borrower{$_} = format_date_in_iso($tempdate);
252	}
253	elsif ( $tempdate =~ /$iso_re/ ) {
254	$borrower{$_} = $tempdate;
255	}
256	else {
257	$borrower{$_} = '';
258	push @missing_criticals, { key => $_, line => $., lineraw => $borrowerline, bad_date => 1 };
259	}
260	}
261	$borrower{dateenrolled} = $today_iso unless $borrower{dateenrolled};
262	$borrower{dateexpiry} = GetExpiryDate( $borrower{categorycode}, $borrower{dateenrolled} )
263	unless $borrower{dateexpiry};
264	my $borrowernumber;
265	my $member;
266	if ( ( $matchpoint eq 'cardnumber' ) && ( $borrower{'cardnumber'} ) ) {
267	$member = GetMember( 'cardnumber' => $borrower{'cardnumber'} );
268	if ($member) {
269	$borrowernumber = $member->{'borrowernumber'};
270	}
271	} elsif ( ($matchpoint eq 'userid') && ($borrower{'userid'}) ) {
272	$member = GetMember( 'userid' => $borrower{'userid'} );
273	if ($member) {
274	$borrowernumber = $member->{'borrowernumber'};
275	}
276	} elsif ($extended) {
277	if (defined($matchpoint_attr_type)) {
278	foreach my $attr (@$patron_attributes) {
279	if ( $attr->{code} eq $matchpoint and $attr->{value} ne '' ) {
280	my @borrowernumbers = $matchpoint_attr_type->get_patrons( $attr->{value} );
281	$borrowernumber = $borrowernumbers[0] if scalar(@borrowernumbers) == 1;
282	last;
283	}
284	}
285	}
286	}
287
288	if ( C4::Members::checkcardnumber( $borrower{cardnumber}, $borrowernumber ) ) {
289	push @errors,
290	{
291	invalid_cardnumber => 1,
292	borrowernumber => $borrowernumber,
293	cardnumber => $borrower{cardnumber}
294	};
295	$invalid++;
296	next;
297	}
298
299	if ($borrowernumber) {
300
301	# borrower exists
302	unless ($overwrite_cardnumber) {
303	$alreadyindb++;
304	$template->param( 'lastalreadyindb' => $borrower{'surname'} . ' / ' . $borrowernumber );
305	next LINE;
306	}
307	$borrower{'borrowernumber'} = $borrowernumber;
308	for my $col ( keys %borrower ) {
309
310	# use values from extant patron unless our csv file includes this column or we provided a default.
311	# FIXME : You cannot update a field with a perl-evaluated false value using the defaults.
312
313	# The password is always encrypted, skip it!
314	next if $col eq 'password';
315
316	unless ( exists( $csvkeycol{$col} ) \|\| $defaults{$col} ) {
317	$borrower{$col} = $member->{$col} if ( $member->{$col} );
318	}
319	}
320	unless ( ModMember(%borrower) ) {
321	$invalid++;
322
323	# untill we have better error trapping, we have no way of knowing why ModMember errored out...
324	push @errors, { unknown_error => 1 };
325	$template->param( 'lastinvalid' => $borrower{'surname'} . ' / ' . $borrowernumber );
326	next LINE;
327	}
328	if ( $borrower{debarred} ) {
329
330	# Check to see if this debarment already exists
331	my $debarrments = GetDebarments(
332	{
333	borrowernumber => $borrowernumber,
334	expiration => $borrower{debarred},
335	comment => $borrower{debarredcomment}
336	}
337	);
338
339	# If it doesn't, then add it!
340	unless (@$debarrments) {
341	AddDebarment(
342	{
343	borrowernumber => $borrowernumber,
344	expiration => $borrower{debarred},
345	comment => $borrower{debarredcomment}
346	}
347	);
348	}
349	}
350	if ($extended) {
351	if ($ext_preserve) {
352	my $old_attributes = GetBorrowerAttributes($borrowernumber);
353	$patron_attributes = extended_attributes_merge( $old_attributes, $patron_attributes )
354	; #TODO: expose repeatable options in template
355	}
356	push @errors, { unknown_error => 1 }
357	unless SetBorrowerAttributes( $borrower{'borrowernumber'}, $patron_attributes );
358	}
359	$overwritten++;
360	$template->param( 'lastoverwritten' => $borrower{'surname'} . ' / ' . $borrowernumber );
361	}
362	else {
363	# FIXME: fixup_cardnumber says to lock table, but the web interface doesn't so this doesn't either.
364	# At least this is closer to AddMember than in members/memberentry.pl
365	if ( !$borrower{'cardnumber'} ) {
366	$borrower{'cardnumber'} = fixup_cardnumber(undef);
367	}
368	if ( $borrowernumber = AddMember(%borrower) ) {
369		122
370	if ( $borrower{debarred} ) {	123	my $feedback = $return->{feedback};
371	AddDebarment(	124	my $errors = $return->{errors};
372	{	125	my $imported = $return->{imported};
373	borrowernumber => $borrowernumber,	126	my $overwritten = $return->{overwritten};
374	expiration => $borrower{debarred},	127	my $alreadyindb = $return->{already_in_db};
375	comment => $borrower{debarredcomment}	128	my $invalid = $return->{invalid};
376	}
377	);
378	}
379		129
380	if ($extended) {	130	my $uploadinfo = $input->uploadInfo($uploadborrowers);
381	SetBorrowerAttributes( $borrowernumber, $patron_attributes );	131	foreach ( keys %$uploadinfo ) {
382	}	132	push @$feedback, { feedback => 1, name => $_, value => $uploadinfo->{$_}, $_ => $uploadinfo->{$_} };
		133	}
383		134
384	if ($set_messaging_prefs) {	135	push @$feedback, { feedback => 1, name => 'filename', value => $uploadborrowers, filename => $uploadborrowers };
385	C4::Members::Messaging::SetMessagingPreferencesFromDefaults(
386	{
387	borrowernumber => $borrowernumber,
388	categorycode => $borrower{categorycode}
389	}
390	);
391	}
392		136
393	$imported++;
394	$template->param( 'lastimported' => $borrower{'surname'} . ' / ' . $borrowernumber );
395	}
396	else {
397	$invalid++;
398	push @errors, { unknown_error => 1 };
399	$template->param( 'lastinvalid' => $borrower{'surname'} . ' / AddMember' );
400	}
401	}
402	}
403	(@errors) and $template->param( ERRORS => \@errors );
404	(@feedback) and $template->param( FEEDBACK => \@feedback );
405	$template->param(	137	$template->param(
406	'uploadborrowers' => 1,	138	uploadborrowers => 1,
407	'imported' => $imported,	139	errors => $errors,
408	'overwritten' => $overwritten,	140	feedback => $feedback,
409	'alreadyindb' => $alreadyindb,	141	imported => $imported,
410	'invalid' => $invalid,	142	overwritten => $overwritten,
411	'total' => $imported + $alreadyindb + $invalid + $overwritten,	143	alreadyindb => $alreadyindb,
		144	invalid => $invalid,
		145	total => $imported + $alreadyindb + $invalid + $overwritten,
412	);	146	);
413		147
414	}	148	}
415	-