View | Details | Raw Unified | Return to bug 42110
Collapse All | Expand All

(-)a/admin/marc_order_accounts.pl (-16 / +15 lines)
Lines 75-96 if ( $op eq 'acct_form' ) { Link Here
75
} else {
75
} else {
76
    if ( $op eq 'cud-save' ) {
76
    if ( $op eq 'cud-save' ) {
77
        my $fields = {
77
        my $fields = {
78
            id                 => scalar $input->param('id'),
78
            id                => scalar $input->param('id'),
79
            description        => scalar $input->param('description'),
79
            description       => scalar $input->param('description'),
80
            vendor_id          => scalar $input->param('vendor_id'),
80
            vendor_id         => scalar $input->param('vendor_id'),
81
            budget_id          => scalar $input->param('budget_id'),
81
            budget_id         => scalar $input->param('budget_id'),
82
            download_directory => scalar $input->param('download_directory'),
82
            matcher_id        => scalar $input->param('matcher'),
83
            matcher_id         => scalar $input->param('matcher'),
83
            overlay_action    => scalar $input->param('overlay_action'),
84
            overlay_action     => scalar $input->param('overlay_action'),
84
            nomatch_action    => scalar $input->param('nomatch_action'),
85
            nomatch_action     => scalar $input->param('nomatch_action'),
85
            parse_items       => scalar $input->param('parse_items'),
86
            parse_items        => scalar $input->param('parse_items'),
86
            item_action       => scalar $input->param('item_action'),
87
            item_action        => scalar $input->param('item_action'),
87
            record_type       => scalar $input->param('record_type'),
88
            record_type        => scalar $input->param('record_type'),
88
            encoding          => scalar $input->param('encoding') || 'UTF-8',
89
            encoding           => scalar $input->param('encoding') || 'UTF-8',
89
            match_field       => scalar $input->param('match_field'),
90
            match_field        => scalar $input->param('match_field'),
90
            match_value       => scalar $input->param('match_value'),
91
            match_value        => scalar $input->param('match_value'),
91
            basket_name_field => scalar $input->param('basket_name_field'),
92
            basket_name_field  => scalar $input->param('basket_name_field'),
92
            file_transport_id => scalar $input->param('file_transport_id') || undef,
93
            file_transport_id  => scalar $input->param('file_transport_id') || undef,
94
        };
93
        };
95
94
96
        if ( scalar $input->param('id') ) {
95
        if ( scalar $input->param('id') ) {
(-)a/api/v1/swagger/definitions/file_transport.yaml (-1 / +1 lines)
Lines 19-25 properties: Link Here
19
    enum:
19
    enum:
20
      - ftp
20
      - ftp
21
      - sftp
21
      - sftp
22
      - file
22
      - local
23
    description: Transport of File transport (optional)
23
    description: Transport of File transport (optional)
24
  auth_mode:
24
  auth_mode:
25
    type: string
25
    type: string
(-)a/installer/data/mysql/atomicupdate/bug_42110.pl (+52 lines)
Line 0 Link Here
1
use Modern::Perl;
2
use Koha::Installer::Output qw(say_warning say_success say_info);
3
4
return {
5
    bug_number  => "42110",
6
    description => "Migrate marc_order_accounts.download_directory to local type file transport",
7
    up          => sub {
8
        my ($args) = @_;
9
        my ( $dbh, $out ) = @$args{qw(dbh out)};
10
11
        if ( column_exists( 'marc_order_accounts', 'download_directory' ) ) {
12
            my $accounts = $dbh->selectall_arrayref(
13
                q{
14
                    SELECT id, description, download_directory
15
                    FROM marc_order_accounts
16
                    WHERE download_directory IS NOT NULL
17
                      AND download_directory != ''
18
                      AND file_transport_id IS NULL
19
                },
20
                { Slice => {} }
21
            );
22
23
            for my $acct ( @{$accounts} ) {
24
                $dbh->do(
25
                    q{
26
                        INSERT INTO file_transports (name, host, port, transport, passive, auth_mode, download_directory, debug)
27
                        VALUES (?, 'localhost', 0, 'local', 1, 'noauth', ?, 0)
28
                    },
29
                    undef,
30
                    "MARC order account: $acct->{description} [$acct->{id}] (Migrated)",
31
                    $acct->{download_directory}
32
                );
33
                my $file_transport_id = $dbh->last_insert_id( undef, undef, 'file_transports', undef );
34
35
                $dbh->do(
36
                    q{ UPDATE marc_order_accounts SET file_transport_id = ? WHERE id = ? },
37
                    undef,
38
                    $file_transport_id,
39
                    $acct->{id}
40
                    ) == 1
41
                    && say_success(
42
                    $out,
43
                    "Migrated download_directory for account '$acct->{description}' to file_transport $file_transport_id"
44
                    );
45
            }
46
47
            $dbh->do(q{ ALTER TABLE `marc_order_accounts` DROP COLUMN `download_directory` });
48
            say_success( $out, "Dropped column 'marc_order_accounts.download_directory'" );
49
        }
50
51
    },
52
};
(-)a/installer/data/mysql/kohastructure.sql (-1 lines)
Lines 4666-4672 CREATE TABLE `marc_order_accounts` ( Link Here
4666
  `description` varchar(250) NOT NULL COMMENT 'description of this account',
4666
  `description` varchar(250) NOT NULL COMMENT 'description of this account',
4667
  `vendor_id` int(11) DEFAULT NULL COMMENT 'vendor id for this account',
4667
  `vendor_id` int(11) DEFAULT NULL COMMENT 'vendor id for this account',
4668
  `budget_id` int(11) DEFAULT NULL COMMENT 'budget id for this account',
4668
  `budget_id` int(11) DEFAULT NULL COMMENT 'budget id for this account',
4669
  `download_directory` mediumtext DEFAULT NULL COMMENT 'download directory for this account',
4670
  `matcher_id` int(11) DEFAULT NULL COMMENT 'the id of the match rule used (matchpoints.matcher_id)',
4669
  `matcher_id` int(11) DEFAULT NULL COMMENT 'the id of the match rule used (matchpoints.matcher_id)',
4671
  `overlay_action` varchar(50) DEFAULT NULL COMMENT 'how to handle duplicate records',
4670
  `overlay_action` varchar(50) DEFAULT NULL COMMENT 'how to handle duplicate records',
4672
  `nomatch_action` varchar(50) DEFAULT NULL COMMENT 'how to handle records where no match is found',
4671
  `nomatch_action` varchar(50) DEFAULT NULL COMMENT 'how to handle records where no match is found',
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/file_transports.tt (-13 / +23 lines)
Lines 146-151 Link Here
146
                        <select name="transport" id="transport" class="required">
146
                        <select name="transport" id="transport" class="required">
147
                            <option value="ftp">FTP</option>
147
                            <option value="ftp">FTP</option>
148
                            <option value="sftp" selected="selected">SFTP</option>
148
                            <option value="sftp" selected="selected">SFTP</option>
149
                            <option value="local">Local</option>
149
                        </select>
150
                        </select>
150
                        <span class="required">Required</span>
151
                        <span class="required">Required</span>
151
                    </li>
152
                    </li>
Lines 296-301 Link Here
296
                            [% ELSE %]
297
                            [% ELSE %]
297
                                <option value="sftp">SFTP</option>
298
                                <option value="sftp">SFTP</option>
298
                            [% END %]
299
                            [% END %]
300
                            [% IF file_transport.transport == 'local' %]
301
                                <option value="local" selected="selected">Local</option>
302
                            [% ELSE %]
303
                                <option value="local">Local</option>
304
                            [% END %]
299
                        </select>
305
                        </select>
300
                        <span class="required">Required</span>
306
                        <span class="required">Required</span>
301
                    </li>
307
                    </li>
Lines 689-707 Link Here
689
            $("#confirm_key_accept_submit").on("click", function (event) {
695
            $("#confirm_key_accept_submit").on("click", function (event) {
690
                event.preventDefault();
696
                event.preventDefault();
691
697
692
                if ($("#add").length > 0) {
698
                var form = $("#add").length > 0 ? $("#add") : $("#edit_save");
693
                    if ($("#add").valid() == true) {
699
                if (form.length > 0) {
694
                        modalChange();
700
                    if (form.valid() == true) {
695
                        $("#confirm_key_accept").modal("show");
701
                        if ($("#transport").val() == "local") {
696
                    } else {
702
                            form.submit();
697
                        $("#confirm_key_accept").modal("hide");
703
                        } else {
698
                    }
704
                            modalChange();
699
                }
705
                            $("#confirm_key_accept").modal("show");
700
706
                        }
701
                if ($("#edit_save").length > 0) {
702
                    if ($("#edit_save").valid() == true) {
703
                        modalChange();
704
                        $("#confirm_key_accept").modal("show");
705
                    } else {
707
                    } else {
706
                        $("#confirm_key_accept").modal("hide");
708
                        $("#confirm_key_accept").modal("hide");
707
                    }
709
                    }
Lines 762-767 Link Here
762
                if (port == 21) $("#port").val("22");
764
                if (port == 21) $("#port").val("22");
763
765
764
                return authModeChange();
766
                return authModeChange();
767
            } else if (transport.val() == "local") {
768
                $("#host").val("").attr("disabled", "disabled");
769
                $("#port").val("").attr("disabled", "disabled");
770
                $("#passive").val("1").attr("disabled", "disabled");
771
                $("#auth_mode").val("noauth").attr("disabled", "disabled");
772
                $("#user_name").val("").attr("disabled", "disabled");
773
                $("#password").val("").attr("disabled", "disabled");
774
                $("#key_file").val("").attr("disabled", "disabled");
765
            }
775
            }
766
        }
776
        }
767
777
(-)a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/marc_order_accounts.tt (-7 lines)
Lines 52-58 Link Here
52
                        <th>Vendor</th>
52
                        <th>Vendor</th>
53
                        <th>Budget</th>
53
                        <th>Budget</th>
54
                        <th>Description</th>
54
                        <th>Description</th>
55
                        <th>Download directory</th>
56
                        <th>Actions</th>
55
                        <th>Actions</th>
57
                    </tr>
56
                    </tr>
58
                    [% FOREACH account IN accounts %]
57
                    [% FOREACH account IN accounts %]
Lines 61-67 Link Here
61
                            <td><a href="/cgi-bin/koha/acquisition/vendors/[% account.vendor_id | uri %]">[% account.vendor.name | html %]</a></td>
60
                            <td><a href="/cgi-bin/koha/acquisition/vendors/[% account.vendor_id | uri %]">[% account.vendor.name | html %]</a></td>
62
                            <td><a href="/cgi-bin/koha/admin/aqbudgets.pl?budget_period_id=[% account.budget.budget_period_id | uri %]">[% account.budget.budget_name | html %]</a></td>
61
                            <td><a href="/cgi-bin/koha/admin/aqbudgets.pl?budget_period_id=[% account.budget.budget_period_id | uri %]">[% account.budget.budget_name | html %]</a></td>
63
                            <td>[% account.description | html %]</td>
62
                            <td>[% account.description | html %]</td>
64
                            <td>[% account.download_directory | html %]</td>
65
                            <td class="actions">
63
                            <td class="actions">
66
                                <a class="btn btn-default btn-xs" href="/cgi-bin/koha/admin/marc_order_accounts.pl?op=acct_form&id=[% account.id | html %]"><i class="fa fa-pencil-alt"></i> Edit</a>
64
                                <a class="btn btn-default btn-xs" href="/cgi-bin/koha/admin/marc_order_accounts.pl?op=acct_form&id=[% account.id | html %]"><i class="fa fa-pencil-alt"></i> Edit</a>
67
                                <a class="btn btn-default btn-xs" href="/cgi-bin/koha/admin/marc_order_accounts.pl?op=delete_acct&id=[% account.id | html %]"><i class="fa fa-trash-can"></i> Delete</a>
65
                                <a class="btn btn-default btn-xs" href="/cgi-bin/koha/admin/marc_order_accounts.pl?op=delete_acct&id=[% account.id | html %]"><i class="fa fa-trash-can"></i> Delete</a>
Lines 135-145 Link Here
135
                            [% END %]
133
                            [% END %]
136
                        </div>
134
                        </div>
137
                    </li>
135
                    </li>
138
                    <li>
139
                        <label for="download_directory">Download directory: </label>
140
                        <input type="text" name="download_directory" id="download_directory" size="20" value="[% account.download_directory | html %]" />
141
                        <div class="hint">The download directory specifies the directory in your Koha installation that should be searched for new files.</div>
142
                    </li>
143
                    <li>
136
                    <li>
144
                        <label for="match_field">Match field: </label>
137
                        <label for="match_field">Match field: </label>
145
                        <input type="text" name="match_field" id="match_field" size="20" value="[% account.match_field | html %]" />
138
                        <input type="text" name="match_field" id="match_field" size="20" value="[% account.match_field | html %]" />
(-)a/misc/cronjobs/marc_ordering_process.pl (-72 / +70 lines)
Lines 23-29 marc_ordering_process.pl - cron script to retrieve MARC files and create order l Link Here
23
23
24
=head1 SYNOPSIS
24
=head1 SYNOPSIS
25
25
26
./marc_ordering_process.pl [-c|--confirm] [-v|--verbose] [--dr|--delete-remote] [--rr|--rename-remote] ext]
26
./marc_ordering_process.pl [-c|--confirm] [-v|--verbose] [-d|--delete] [-r|--rename ext]
27
27
28
or, in crontab:
28
or, in crontab:
29
# Once every day
29
# Once every day
Lines 31-38 or, in crontab: Link Here
31
31
32
=head1 DESCRIPTION
32
=head1 DESCRIPTION
33
33
34
This script searches for new MARC files in an SFTP location
34
This script searches for new MARC files via configured file transports,
35
If there are new files, it stages those files, adds bilbios/items and creates order lines
35
stages those files, adds biblios/items and creates order lines.
36
36
37
=head1 OPTIONS
37
=head1 OPTIONS
38
38
Lines 48-62 Without this parameter no changes will be made Link Here
48
48
49
=item B<-d|--delete>
49
=item B<-d|--delete>
50
50
51
Delete the file once it has been processed
51
Delete the remote source file once it has been processed
52
52
53
=item B<--dr|--delete-remote>
53
=item B<-r|--rename ext>
54
54
55
Delete the remote file once it has been downloaded
55
Rename the remote source file once it has been processed, adding the given file extension
56
56
57
=item B<-rr|--rename-remove ext>
57
=item B<-a|--archive dir>
58
58
59
Rename the remote file once it has been downloaded, adding the given file extension
59
Copy downloaded files into the specified local archive directory after processing
60
60
61
=back
61
=back
62
62
Lines 65-71 Rename the remote file once it has been downloaded, adding the given file extens Link Here
65
use Modern::Perl;
65
use Modern::Perl;
66
use Pod::Usage   qw( pod2usage );
66
use Pod::Usage   qw( pod2usage );
67
use Getopt::Long qw( GetOptions );
67
use Getopt::Long qw( GetOptions );
68
use File::Copy   qw( copy move );
68
use File::Copy   qw( copy );
69
use File::Temp   qw( tempdir );
69
70
70
use Koha::Script -cron;
71
use Koha::Script -cron;
71
use Koha::MarcOrder;
72
use Koha::MarcOrder;
Lines 76-89 use C4::Log qw( cronlogaction ); Link Here
76
my $command_line_options = join( " ", @ARGV );
77
my $command_line_options = join( " ", @ARGV );
77
cronlogaction( { info => $command_line_options } );
78
cronlogaction( { info => $command_line_options } );
78
79
79
my ( $help, $verbose, $confirm, $delete, $rename_ext, $delete_remote );
80
my ( $help, $verbose, $confirm, $delete, $rename_ext, $archive_dir );
80
GetOptions(
81
GetOptions(
81
    'h|help'             => \$help,
82
    'h|help'      => \$help,
82
    'v|verbose'          => \$verbose,
83
    'v|verbose'   => \$verbose,
83
    'c|confirm'          => \$confirm,
84
    'c|confirm'   => \$confirm,
84
    'd|delete'           => \$delete,
85
    'd|delete'    => \$delete,
85
    'dr|delete-remote'   => \$delete_remote,
86
    'r|rename=s'  => \$rename_ext,
86
    'rr|rename-remote=s' => \$rename_ext,
87
    'a|archive=s' => \$archive_dir,
87
) || pod2usage(1);
88
) || pod2usage(1);
88
89
89
pod2usage(0) if $help;
90
pod2usage(0) if $help;
Lines 104-173 if ( scalar(@accounts) == 0 ) { Link Here
104
my $valid_file_extensions = qr/\.(mrc|marcxml|mrk)$/i;
105
my $valid_file_extensions = qr/\.(mrc|marcxml|mrk)$/i;
105
106
106
foreach my $acct (@accounts) {
107
foreach my $acct (@accounts) {
108
    my $file_transport = $acct->file_transport;
109
    unless ($file_transport) {
110
        say "No file transport configured for account: " . $acct->description if $verbose;
111
        next;
112
    }
113
114
    my $working_dir = $file_transport->download_directory;
115
    unless ($working_dir) {
116
        say "No download directory configured for file transport: " . $file_transport->id if $verbose;
117
        next;
118
    }
119
107
    if ($verbose) {
120
    if ($verbose) {
108
        say sprintf "Starting MARC ordering process for %s", $acct->vendor->name;
121
        say sprintf "Starting MARC ordering process for %s", $acct->vendor->name;
109
        say sprintf "Looking for new files in %s",           $acct->download_directory;
122
        say sprintf "Looking for new files in %s",           $working_dir;
110
    }
123
    }
111
124
112
    my $working_dir = $acct->download_directory;
125
    unless ( $file_transport->connect() ) {
113
126
        say "Failed to connect to file transport: " . $file_transport->id;
114
    my $file_transport = $acct->file_transport;
127
        next;
115
    if ($file_transport) {
128
    }
116
        if ( $file_transport->connect() ) {
117
            my $download_dir = $file_transport->download_directory;
118
129
119
            my $success = $download_dir ? $file_transport->change_directory($download_dir) : 1;
130
    my $success = $file_transport->change_directory($working_dir);
120
            if ( $download_dir && !$success ) {
131
    if ( !$success ) {
121
                say "Failed to change to download directory: $download_dir";
132
        say "Failed to change to download directory: $working_dir";
122
            } else {
133
        next;
134
    }
123
135
124
                # Get file list
136
    # Get file list
125
                my $file_list = $file_transport->list_files();
137
    my $file_list = $file_transport->list_files();
126
                if ($file_list) {
138
    unless ($file_list) {
127
139
        say "Failed to get file list from transport";
128
                    # Process files matching our criteria
140
        next;
129
                    foreach my $file ( @{$file_list} ) {
141
    }
130
                        my $filename = $file->{filename};
131
132
                        if ( $filename =~ $valid_file_extensions ) {
133
134
                            my $local_file = "$working_dir/$filename";
135
136
                            # Download the file
137
                            if ( $file_transport->download_file( $filename, $local_file ) ) {
138
                                if ($rename_ext) {
139
                                    $file_transport->rename_file( $filename, "$filename.$rename_ext" );
140
                                } elsif ($delete_remote) {
141
                                    $file_transport->delete_file($filename);
142
                                }
143
                            } else {
144
                                say "Failed to download file: $filename";
145
                            }
146
                        }
147
                    }
148
149
                } else {
150
                    say "Failed to get file list from transport";
151
                }
152
142
153
            }
143
    my @files;
154
        } else {
144
    foreach my $file ( @{$file_list} ) {
155
            say "Failed to connect to file transport: " . $file_transport->id;
145
        my $filename = $file->{filename};
156
        }
146
        next unless $filename =~ $valid_file_extensions;
147
        push @files, $filename;
157
    }
148
    }
158
149
159
    opendir my $dir, $working_dir or die "Can't open filepath";
160
    my @files = grep { /$valid_file_extensions/ } readdir $dir;
161
    closedir $dir;
162
    print "No new files found\n" if scalar(@files) == 0;
150
    print "No new files found\n" if scalar(@files) == 0;
163
151
164
    my $files_processed = 0;
152
    my $files_processed = 0;
153
    my $local_dir       = tempdir( CLEANUP => 1 );
165
154
166
    foreach my $filename (@files) {
155
    foreach my $filename (@files) {
167
        my $full_path = "$working_dir/$filename";
156
        my $local_file = "$local_dir/$filename";
168
        my $args      = {
157
158
        unless ( $file_transport->download_file( $filename, $local_file ) ) {
159
            say "Failed to download file: $filename";
160
            next;
161
        }
162
163
        my $args = {
169
            filename => $filename,
164
            filename => $filename,
170
            filepath => $full_path,
165
            filepath => $local_file,
171
            profile  => $acct,
166
            profile  => $acct,
172
            agent    => 'cron'
167
            agent    => 'cron'
173
        };
168
        };
Lines 182-194 foreach my $acct (@accounts) { Link Here
182
            if ( $result->{success} ) {
177
            if ( $result->{success} ) {
183
                $files_processed++;
178
                $files_processed++;
184
                say sprintf "Successfully processed file: %s", $filename if $verbose;
179
                say sprintf "Successfully processed file: %s", $filename if $verbose;
180
                if ($archive_dir) {
181
                    mkdir $archive_dir unless -d $archive_dir;
182
                    say sprintf "Archiving file: %s to %s", $filename, $archive_dir if $verbose;
183
                    copy( $local_file, "$archive_dir/$filename" );
184
                }
185
                if ($delete) {
185
                if ($delete) {
186
                    say sprintf "Deleting processed file: %s", $filename if $verbose;
186
                    say sprintf "Deleting file: %s", $filename if $verbose;
187
                    unlink $full_path;
187
                    $file_transport->delete_file($filename);
188
                } else {
188
                } elsif ($rename_ext) {
189
                    mkdir "$working_dir/archive" unless -d "$working_dir/archive";
189
                    say sprintf "Renaming file: %s to %s.%s", $filename, $filename, $rename_ext if $verbose;
190
                    say sprintf "Moving file to archive: %s", $filename if $verbose;
190
                    $file_transport->rename_file( $filename, "$filename.$rename_ext" );
191
                    move( $full_path, "$working_dir/archive/$filename" );
192
                }
191
                }
193
            } else {
192
            } else {
194
                say sprintf "Error processing file: %s", $filename        if $verbose;
193
                say sprintf "Error processing file: %s", $filename        if $verbose;
195
- 

Return to bug 42110