@@ -, +, @@ the renamed scripts misc/stage_file.pl and misc/commit_file.pl is attached to bug 7475) and download it to your server the name of the file you saved in step 1): > misc/stage_file.pl --file --authorities with the batch number you made note of in step 3): > misc/commit_file.pl --batch-number > misc/commit_file.pl --batch-number --revert --- C4/ImportBatch.pm | 304 +++++++++++++------- C4/Matcher.pm | 73 ++++- admin/matching-rules.pl | 12 +- .../data/mysql/atomicupdate/importauthorities.pl | 19 ++ installer/data/mysql/kohastructure.sql | 15 + .../prog/en/modules/admin/matching-rules.tt | 11 + misc/{commit_biblios_file.pl => commit_file.pl} | 54 +++- misc/cronjobs/import_webservice_batch.pl | 2 +- misc/{stage_biblios_file.pl => stage_file.pl} | 84 +++--- svc/import_bib | 2 +- tools/manage-marc-import.pl | 4 +- 11 files changed, 410 insertions(+), 170 deletions(-) create mode 100755 installer/data/mysql/atomicupdate/importauthorities.pl rename misc/{commit_biblios_file.pl => commit_file.pl} (63%) rename misc/{stage_biblios_file.pl => stage_file.pl} (67%) --- a/C4/ImportBatch.pm +++ a/C4/ImportBatch.pm @@ -1,6 +1,6 @@ package C4::ImportBatch; -# Copyright (C) 2007 LibLime +# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services # # This file is part of Koha. # @@ -40,14 +40,16 @@ BEGIN { GetImportRecordMarcXML AddImportBatch GetImportBatch + AddAuthorityToBatch AddBiblioToBatch AddItemsToImportBiblio + ModAuthorityInBatch ModBiblioInBatch BatchStageMarcRecords - BatchFindBibDuplicates - BatchCommitBibRecords - BatchRevertBibRecords + BatchFindDuplicates + BatchCommitRecords + BatchRevertRecords CleanBatch GetAllImportBatches @@ -272,10 +274,43 @@ sub ModBiblioInBatch { } +=head2 AddAuthToBatch + + my $import_record_id = AddAuthToBatch($batch_id, $record_sequence, + $marc_record, $encoding, $z3950random, $update_counts); + +=cut + +sub AddAuthToBatch { + my $batch_id = shift; + my $record_sequence = shift; + my $marc_record = shift; + my $encoding = shift; + my $z3950random = shift; + my $update_counts = @_ ? shift : 1; + + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random); + _update_batch_record_counts($batch_id) if $update_counts; + return $import_record_id; +} + +=head2 ModAuthInBatch + + ModAuthInBatch($import_record_id, $marc_record); + +=cut + +sub ModAuthInBatch { + my ($import_record_id, $marc_record) = @_; + + _update_import_record_marc($import_record_id, $marc_record); + +} + =head2 BatchStageMarcRecords ($batch_id, $num_records, $num_items, @invalid_records) = - BatchStageMarcRecords($encoding, $marc_records, $file_name, + BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name, $comments, $branch_code, $parse_items, $leave_as_staging, $progress_interval, $progress_callback); @@ -283,6 +318,7 @@ sub ModBiblioInBatch { =cut sub BatchStageMarcRecords { + my $record_type = shift; my $encoding = shift; my $marc_records = shift; my $file_name = shift; @@ -338,10 +374,14 @@ sub BatchStageMarcRecords { push @invalid_records, $marc_blob; } else { $num_valid++; - $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); - if ($parse_items) { - my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); - $num_items += scalar(@import_items_ids); + if ($record_type eq 'biblio') { + $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); + if ($parse_items) { + my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); + $num_items += scalar(@import_items_ids); + } + } elsif ($record_type eq 'auth') { + $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); } } } @@ -392,9 +432,9 @@ sub AddItemsToImportBiblio { return @import_items_ids; } -=head2 BatchFindBibDuplicates +=head2 BatchFindDuplicates - my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, + my $num_with_matches = BatchFindDuplicates($batch_id, $matcher, $max_matches, $progress_interval, $progress_callback); Goes through the records loaded in the batch and attempts to @@ -412,7 +452,7 @@ singular argument. =cut -sub BatchFindBibDuplicates { +sub BatchFindDuplicates { my $batch_id = shift; my $matcher = shift; my $max_matches = @_ ? shift : 10; @@ -430,9 +470,10 @@ sub BatchFindBibDuplicates { my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, marc + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, marc FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); my $num_with_matches = 0; @@ -460,15 +501,15 @@ sub BatchFindBibDuplicates { return $num_with_matches; } -=head2 BatchCommitBibRecords +=head2 BatchCommitRecords - my ($num_added, $num_updated, $num_items_added, $num_items_errored, - $num_ignored) = BatchCommitBibRecords($batch_id, $framework, - $progress_interval, $progress_callback); + my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + BatchCommitRecords($batch_id, $framework, + $progress_interval, $progress_callback); =cut -sub BatchCommitBibRecords { +sub BatchCommitRecords { my $batch_id = shift; my $framework = shift; @@ -483,25 +524,29 @@ sub BatchCommitBibRecords { $progress_interval = 0 unless 'CODE' eq ref $progress_callback; } + my $record_type; my $num_added = 0; my $num_updated = 0; my $num_items_added = 0; my $num_items_errored = 0; my $num_ignored = 0; # commit (i.e., save, all records in the batch) - # FIXME biblio only at the moment SetImportBatchStatus('importing'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $item_action = GetImportBatchItemAction($batch_id); + my $item_tag; + my $item_subfield; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); my $rec_num = 0; while (my $rowref = $sth->fetchrow_hashref) { + $record_type = $rowref->{'record_type'}; $rec_num++; if ($progress_interval and (0 == ($rec_num % $progress_interval))) { &$progress_callback($rec_num); @@ -513,67 +558,87 @@ sub BatchCommitBibRecords { my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'}); - # remove any item tags - rely on BatchCommitItems - my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); - foreach my $item_field ($marc_record->field($item_tag)) { - $marc_record->delete_field($item_field); + if ($record_type eq 'biblio') { + # remove any item tags - rely on BatchCommitItems + ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); + foreach my $item_field ($marc_record->field($item_tag)) { + $marc_record->delete_field($item_field); + } } - # decide what what to do with the bib and item records - my ($bib_result, $item_result, $bib_match) = + my ($record_result, $item_result, $record_match) = _get_commit_action($overlay_action, $nomatch_action, $item_action, - $rowref->{'overlay_status'}, $rowref->{'import_record_id'}); + $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type); - if ($bib_result eq 'create_new') { + my $recordid; + my $query; + if ($record_result eq 'create_new') { $num_added++; - my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, $framework); - my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth->execute($biblionumber, $rowref->{'import_record_id'}); - $sth->finish(); - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($record_type eq 'biblio') { + my $biblioitemnumber; + ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } + } else { + my $authid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record)); + $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; } + my $sth = $dbh->prepare_cached($query); + $sth->execute($recordid, $rowref->{'import_record_id'}); + $sth->finish(); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } elsif ($bib_result eq 'replace') { + } elsif ($record_result eq 'replace') { $num_updated++; - my $biblionumber = $bib_match; - my ($count, $oldbiblio) = GetBiblio($biblionumber); - my $oldxml = GetXmlBiblio($biblionumber); - - # remove item fields so that they don't get - # added again if record is reverted - my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}); - foreach my $item_field ($old_marc->field($item_tag)) { - $old_marc->delete_field($item_field); - } + $recordid = $record_match; + my $oldxml; + if ($record_type eq 'biblio') { + my ($count, $oldbiblio) = GetBiblio($recordid); + $oldxml = GetXmlBiblio($recordid); + + # remove item fields so that they don't get + # added again if record is reverted + my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}); + foreach my $item_field ($old_marc->field($item_tag)) { + $old_marc->delete_field($item_field); + } + $oldxml = $old_marc->as_xml(); + + ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'}); + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } + } else { + my $oldxml = GetAuthorityXML($recordid); - ModBiblio($marc_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record)); + $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; + } my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?"); - $sth->execute($old_marc->as_xml(), $rowref->{'import_record_id'}); + $sth->execute($oldxml, $rowref->{'import_record_id'}); $sth->finish(); - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + my $sth2 = $dbh->prepare_cached($query); + $sth2->execute($recordid, $rowref->{'import_record_id'}); $sth2->finish(); - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; - } SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } elsif ($bib_result eq 'ignore') { + } elsif ($record_result eq 'ignore') { $num_ignored++; - my $biblionumber = $bib_match; - if (defined $biblionumber and $item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); $num_items_added += $bib_items_added; $num_items_errored += $bib_items_errored; # still need to record the matched biblionumber so that the # items can be reverted my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + $sth2->execute($recordid, $rowref->{'import_record_id'}); SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); } SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); @@ -632,62 +697,82 @@ sub BatchCommitItems { return ($num_items_added, $num_items_errored); } -=head2 BatchRevertBibRecords +=head2 BatchRevertRecords my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, - $num_ignored) = BatchRevertBibRecords($batch_id); + $num_ignored) = BatchRevertRecords($batch_id); =cut -sub BatchRevertBibRecords { +sub BatchRevertRecords { my $batch_id = shift; + my $record_type; my $num_deleted = 0; my $num_errors = 0; my $num_reverted = 0; - my $num_items_deleted = 0; my $num_ignored = 0; + my $num_items_deleted = 0; # commit (i.e., save, all records in the batch) - # FIXME biblio only at the moment SetImportBatchStatus('reverting'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); while (my $rowref = $sth->fetchrow_hashref) { + $record_type = $rowref->{'record_type'}; if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') { $num_ignored++; next; } - my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); + my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); - if ($bib_result eq 'delete') { - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - my $error = DelBiblio($rowref->{'matched_biblionumber'}); + if ($record_result eq 'delete') { + my $error = undef; + if ($record_type eq 'biblio') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + $error = DelBiblio($rowref->{'matched_biblionumber'}); + } else { + my $deletedauthid = DelAuthority($rowref->{'matched_authid'}); + } if (defined $error) { $num_errors++; } else { $num_deleted++; SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - } elsif ($bib_result eq 'restore') { + } elsif ($record_result eq 'restore') { $num_reverted++; my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}); - my $biblionumber = $rowref->{'matched_biblionumber'}; - my ($count, $oldbiblio) = GetBiblio($biblionumber); - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + if ($record_type eq 'biblio') { + my $biblionumber = $rowref->{'matched_biblionumber'}; + my ($count, $oldbiblio) = GetBiblio($biblionumber); + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + } else { + my $authid = $rowref->{'matched_authid'}; + ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record)); + } SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); - } elsif ($bib_result eq 'ignore') { - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + } elsif ($record_result eq 'ignore') { + if ($record_type eq 'biblio') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + } SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"); + my $query; + if ($record_type eq 'biblio') { + $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; + } else { + $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?"; + } + my $sth2 = $dbh->prepare_cached($query); $sth2->execute($rowref->{'import_record_id'}); } @@ -1305,26 +1390,45 @@ sub _update_batch_record_counts { } sub _get_commit_action { - my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_; + my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_; - my ($bib_result, $bib_match, $item_result); - - if ($overlay_status ne 'no_match') { - $bib_match = GetBestRecordMatch($import_record_id); - if ($overlay_action eq 'replace') { - $bib_result = defined($bib_match) ? 'replace' : 'create_new'; - } elsif ($overlay_action eq 'create_new') { - $bib_result = 'create_new'; - } elsif ($overlay_action eq 'ignore') { - $bib_result = 'ignore'; - } - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; - } else { - $bib_result = $nomatch_action; - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; - } + if ($record_type eq 'biblio') { + my ($bib_result, $bib_match, $item_result); + + if ($overlay_status ne 'no_match') { + $bib_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $bib_result = defined($bib_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $bib_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $bib_result = 'ignore'; + } + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; + } else { + $bib_result = $nomatch_action; + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; + } + return ($bib_result, $item_result, $bib_match); + } else { # must be auths + my ($auth_result, $auth_match); + + if ($overlay_status ne 'no_match') { + $auth_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $auth_result = defined($auth_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $auth_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $auth_result = 'ignore'; + } + } else { + $auth_result = $nomatch_action; + } + + return ($auth_result, undef, $auth_match); - return ($bib_result, $item_result, $bib_match); + } } sub _get_revert_action { --- a/C4/Matcher.pm +++ a/C4/Matcher.pm @@ -1,6 +1,6 @@ package C4::Matcher; -# Copyright (C) 2007 LibLime +# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services # # This file is part of Koha. # @@ -22,8 +22,6 @@ use warnings; use C4::Context; use MARC::Record; -use C4::Search; -use C4::Biblio; use vars qw($VERSION); @@ -384,6 +382,20 @@ sub delete { $sth->execute($matcher_id); # relying on cascading deletes to clean up everything } +=head2 record_type + + $matcher->record_type('biblio'); + my $record_type = $matcher->record_type(); + +Accessor method. + +=cut + +sub record_type { + my $self = shift; + @_ ? $self->{'record_type'} = shift : $self->{'record_type'}; +} + =head2 threshold $matcher->threshold(1000); @@ -582,7 +594,7 @@ sub add_simple_required_check { ); } -=head2 find_matches +=head2 get_matches my @matches = $matcher->get_matches($marc_record, $max_matches); foreach $match (@matches) { @@ -618,9 +630,34 @@ sub get_matches { my @source_keys = _get_match_keys($source_record, $matchpoint); next if scalar(@source_keys) == 0; # build query - my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys); - # FIXME only searching biblio index at the moment - my ($error, $searchresults, $total_hits) = SimpleSearch($query, 0, $max_matches); + my $query; + my $error; + my $searchresults; + my $total_hits; + if ($self->{'record_type'} eq 'biblio') { + $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys); +# FIXME only searching biblio index at the moment + require C4::Search; + ($error, $searchresults, $total_hits) = C4::Search::SimpleSearch($query, 0, $max_matches); + } elsif ($self->{'record_type'} eq 'authority') { + my $authresults; + my @marclist; + my @and_or; + my @excluding = []; + my @operator; + my @value; + foreach my $key (@source_keys) { + push @marclist, $matchpoint->{'index'}; + push @and_or, 'or'; + push @operator, 'exact'; + push @value, $key; + } + require C4::AuthoritiesMarc; + ($searchresults, $total_hits) = C4::AuthoritiesMarc::SearchAuthorities( + \@marclist, \@and_or, \@excluding, \@operator, + \@value, 0, 20, undef, 'AuthidAsc', 1 + ); + } if (defined $error ) { warn "search failed ($query) $error"; @@ -639,13 +676,20 @@ sub get_matches { keys %matches; my @results = (); - foreach my $marcblob (keys %matches) { - my $target_record = MARC::Record->new_from_usmarc($marcblob); - my $result = TransformMarcToKoha(C4::Context->dbh, $target_record, ''); - # FIXME - again, bibliospecific - # also, can search engine be induced to give just the number in the first place? - my $record_number = $result->{'biblionumber'}; - push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} }; + if ($self->{'record_type'} eq 'biblio') { + require C4::Biblio; + foreach my $marcblob (keys %matches) { + my $target_record = MARC::Record->new_from_usmarc($marcblob); + my $record_number; + my $result = C4::Biblio::TransformMarcToKoha(C4::Context->dbh, $target_record, ''); + $record_number = $result->{'biblionumber'}; + push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} }; + } + } elsif ($self->{'record_type'} eq 'authority') { + require C4::AuthoritiesMarc; + foreach my $authid (keys %matches) { + push @results, { 'record_id' => $authid, 'score' => $matches{$authid} }; + } } @results = sort { $b->{'score'} cmp $a->{'score'} } @results; if (scalar(@results) > $max_matches) { @@ -673,6 +717,7 @@ sub dump { $result->{'matcher_id'} = $self->{'id'}; $result->{'code'} = $self->{'code'}; $result->{'description'} = $self->{'description'}; + $result->{'record_type'} = $self->{'record_type'}; $result->{'matchpoints'} = []; foreach my $matchpoint (@{ $self->{'matchpoints'} }) { --- a/admin/matching-rules.pl +++ a/admin/matching-rules.pl @@ -92,9 +92,10 @@ sub add_matching_rule_form { sub add_update_matching_rule { my $template = shift; my $matcher_id = shift; + my $record_type = $input->param('record_type') || 'biblio'; # do parsing - my $matcher = C4::Matcher->new('biblio', 1000); # FIXME biblio only for now + my $matcher = C4::Matcher->new($record_type, 1000); $matcher->code($input->param('code')); $matcher->description($input->param('description')); $matcher->threshold($input->param('threshold')); @@ -203,10 +204,11 @@ sub edit_matching_rule_form { my $matcher = C4::Matcher->fetch($matcher_id); - $template->param(matcher_id => $matcher_id); - $template->param(code => $matcher->code()); - $template->param(description => $matcher->description()); - $template->param(threshold => $matcher->threshold()); + $template->{VARS}->{'matcher_id'} = $matcher_id; + $template->{VARS}->{'code'} = $matcher->code(); + $template->{VARS}->{'description'} = $matcher->description(); + $template->{VARS}->{'threshold'} = $matcher->threshold(); + $template->{VARS}->{'record_type'} = $matcher->record_type(); my $matcher_info = $matcher->dump(); my @matchpoints = (); --- a/installer/data/mysql/atomicupdate/importauthorities.pl +++ a/installer/data/mysql/atomicupdate/importauthorities.pl @@ -0,0 +1,19 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use C4::Context; +my $dbh = C4::Context->dbh; + +$dbh->do( +q|CREATE TABLE `import_auths` ( + `import_record_id` int(11) NOT NULL, + `matched_authid` int(11) default NULL, + `control_number` varchar(25) default NULL, + `original_source` varchar(25) default NULL, + CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`) + REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE, + KEY `matched_authid` (`matched_authid`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;| +); +print "Upgrade done (Added support for staging authorities)\n"; --- a/installer/data/mysql/kohastructure.sql +++ a/installer/data/mysql/kohastructure.sql @@ -909,6 +909,21 @@ CREATE TABLE `import_record_matches` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- +-- Table structure for table `import_auths` +-- + +DROP TABLE IF EXISTS `import_auths`; +CREATE TABLE `import_auths` ( + `import_record_id` int(11) NOT NULL, + `matched_authid` int(11) default NULL, + `control_number` varchar(25) default NULL, + `original_source` varchar(25) default NULL, + CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`) + REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE, + KEY `matched_authid` (`matched_authid`), +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- -- Table structure for table `import_biblios` -- --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt +++ a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt @@ -220,6 +220,17 @@ function CheckRuleForm(f) { Required +
  • + Required +
  • --- a/misc/commit_biblios_file.pl +++ a/misc/commit_biblios_file.pl @@ -18,11 +18,13 @@ $| = 1; # command-line parameters my $batch_number = ""; my $list_batches = 0; +my $revert = 0; my $want_help = 0; my $result = GetOptions( 'batch-number:s' => \$batch_number, 'list-batches' => \$list_batches, + 'revert' => \$revert, 'h|help' => \$want_help ); @@ -45,9 +47,15 @@ $dbh->{AutoCommit} = 0; if ($batch_number =~ /^\d+$/ and $batch_number > 0) { my $batch = GetImportBatch($batch_number); die "$0: import batch $batch_number does not exist in database\n" unless defined $batch; - die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" - unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted"; - process_batch($batch_number); + if ($revert) { + die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" + unless $batch->{'import_status'} eq "imported"; + revert_batch($batch_number); + } else { + die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" + unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted"; + process_batch($batch_number); + } $dbh->commit(); } else { die "$0: please specify a numeric batch ID\n"; @@ -74,8 +82,8 @@ sub process_batch { my ($import_batch_id) = @_; print "... importing MARC records -- please wait\n"; - my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = - BatchCommitBibRecords($import_batch_id, '', 100, \&print_progress_and_commit); + my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + BatchCommitRecords($import_batch_id, '', 100, \&print_progress_and_commit); print "... finished importing MARC records\n"; print <<_SUMMARY_; @@ -83,17 +91,40 @@ sub process_batch { MARC record import report ---------------------------------------- Batch number: $import_batch_id -Number of new bibs added: $num_added -Number of bibs replaced: $num_updated -Number of bibs ignored: $num_ignored +Number of new records added: $num_added +Number of records replaced: $num_updated +Number of records ignored: $num_ignored Number of items added: $num_items_added Number of items ignored: $num_items_errored -Note: an item is ignored if its barcode is a +Note: an item is ignored if its barcode is a duplicate of one already in the database. _SUMMARY_ } +sub revert_batch { + my ($import_batch_id) = @_; + + print "... reverting batch -- please wait\n"; + my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = + BatchRevertRecords($import_batch_id, 100, \&print_progress_and_commit); + print "... finished reverting batch\n"; + + print <<_SUMMARY_; + +MARC record import report +---------------------------------------- +Batch number: $import_batch_id +Number of records deleted: $num_deleted +Number of errors: $num_errors +Number of records reverted: $num_reverted +Number of records ignored: $num_ignored +Number of items added: $num_items_deleted + +_SUMMARY_ +} + + sub print_progress_and_commit { my $recs = shift; print "... processed $recs records\n"; @@ -106,7 +137,7 @@ $0: import a batch of staged MARC records into database. Use this batch job to complete the import of a batch of MARC records that was staged either by the batch job -stage_biblios_file.pl or by the Koha Tools option +stage_file.pl or by the Koha Tools option "Stage MARC Records for Import". Parameters: @@ -114,6 +145,7 @@ Parameters: to import --list-batches print a list of record batches available to commit - --help or -h show this message. + --revert revert a batch instead of importing it + --help or -h show this message. _USAGE_ } --- a/misc/cronjobs/import_webservice_batch.pl +++ a/misc/cronjobs/import_webservice_batch.pl @@ -54,4 +54,4 @@ EOF my $batch_ids = GetStagedWebserviceBatches() or exit; $framework ||= ''; -BatchCommitBibRecords($_, $framework) foreach @$batch_ids; +BatchCommitRecords($_, $framework) foreach @$batch_ids; --- a/misc/stage_biblios_file.pl +++ a/misc/stage_biblios_file.pl @@ -4,6 +4,7 @@ # # Copyright (C) 2007 LibLime # Parts Copyright BSZ 2011 +# Parts Copyright C & P Bibliography Services 2012 # # Koha is free software; you can redistribute it and/or modify it under the # terms of the GNU General Public License as published by the Free Software @@ -35,8 +36,10 @@ use Getopt::Long; $| = 1; # command-line parameters +my $record_type = "biblio"; my $encoding = ""; -my $match_bibs = 0; +my $authorities = 0; +my $match = 0; my $add_items = 0; my $input_file = ""; my $batch_comment = ""; @@ -46,13 +49,16 @@ my $no_replace ; my $result = GetOptions( 'encoding:s' => \$encoding, 'file:s' => \$input_file, - 'match-bibs:s' => \$match_bibs, + 'match|match-bibs:s' => \$match, 'add-items' => \$add_items, 'no-replace' => \$no_replace, 'comment:s' => \$batch_comment, + 'authorities' => \$authorities, 'h|help' => \$want_help ); +$record_type = 'biblio' if ($authorities); + if ($encoding eq "") { $encoding = "utf8"; } @@ -68,13 +74,13 @@ unless (-r $input_file) { my $dbh = C4::Context->dbh; $dbh->{AutoCommit} = 0; -process_batch($input_file, $match_bibs, $add_items, $batch_comment); +process_batch($input_file, $record_type, $match, $add_items, $batch_comment); $dbh->commit(); exit 0; sub process_batch { - my ($input_file, $match_bibs, $add_items, $batch_comment) = @_; + my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_; open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; my $marc_records = ""; @@ -92,50 +98,53 @@ sub process_batch { close IN; print "... staging MARC records -- please wait\n"; - my ($batch_id, $num_valid, $num_items, @import_errors) = - BatchStageMarcRecords($encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0, + my ($batch_id, $num_valid_records, $num_items, @import_errors) = + BatchStageMarcRecords($record_type, $encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0, 100, \&print_progress_and_commit); print "... finished staging MARC records\n"; my $num_with_matches = 0; - if ($match_bibs) { - my $matcher = C4::Matcher->fetch($match_bibs) ; - if (! defined $matcher) { - $matcher = C4::Matcher->new('biblio'); + if ($match) { + my $matcher = C4::Matcher->fetch($match) ; + if (defined $matcher) { + SetImportBatchMatcher($batch_id, $match); + } elsif ($record_type eq 'biblio') { + $matcher = C4::Matcher->new($record_type); $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); - $matcher->add_simple_required_check('245', 'a', -1, 0, '', + $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, ''); - } else { - SetImportBatchMatcher($batch_id, $match_bibs); } # set default record overlay behavior SetImportBatchOverlayAction($batch_id, ($no_replace) ? 'ignore' : 'replace'); SetImportBatchNoMatchAction($batch_id, 'create_new'); SetImportBatchItemAction($batch_id, 'always_add'); print "... looking for matches with records already in database\n"; - $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); + $num_with_matches = BatchFindDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); print "... finished looking for matches\n"; } - my $num_invalid_bibs = scalar(@import_errors); + my $num_invalid_records = scalar(@import_errors); print <<_SUMMARY_; MARC record staging report ------------------------------------ -Input file: $input_file -Number of input bibs: $num_input_records -Number of valid bibs: $num_valid -Number of invalid bibs: $num_invalid_bibs +Input file: $input_file +Record type: $record_type +Number of input records: $num_input_records +Number of valid records: $num_valid_records +Number of invalid records: $num_invalid_records _SUMMARY_ - if ($match_bibs) { - print "Number of bibs matched: $num_with_matches\n"; + if ($match) { + print "Number of records matched: $num_with_matches\n"; } else { - print "Incoming bibs not matched against existing bibs (--match-bibs option not supplied)\n"; + print "Incoming records not matched against existing records (--match option not supplied)\n"; } - if ($add_items) { - print "Number of items parsed: $num_items\n"; - } else { - print "No items parsed (--add-items option not supplied)\n"; + if ($record_type eq 'biblio') { + if ($add_items) { + print "Number of items parsed: $num_items\n"; + } else { + print "No items parsed (--add-items option not supplied)\n"; + } } print "\n"; @@ -151,31 +160,34 @@ sub print_progress_and_commit { sub print_usage { print <<_USAGE_; -$0: stage MARC bib file into reservoir. +$0: stage MARC file into reservoir. -Use this batch job to load a file of MARC bibliographic records -(with optional item information) into the Koha reservoir. +Use this batch job to load a file of MARC bibliographic +(with optional item information) or authority records into +the Koha reservoir. After running this program to stage your file, you can use -either the batch job commit_biblios_file.pl or the Koha +either the batch job commit_file.pl or the Koha Tools option "Manage Staged MARC Records" to load the records into the main Koha database. Parameters: --file name of input MARC bib file + --authorities stage authority records instead of bibs --encoding encoding of MARC records, default is utf8. Other possible options are: MARC-8, ISO_5426, ISO_6937, ISO_8859-1, EUC-KR - --match-bibs use this option to match bibs - in the file with bibs already in + --match use this option to match records + in the file with records already in the database for future overlay. - If isn't defined, a default - MARC21 ISBN & title match rule will be applied. + If isn't defined, a default + MARC21 ISBN & title match rule will be applied + for bib imports. --add-items use this option to specify that item data is embedded in the MARC bibs and should be parsed. - --no-replace overlay action for bib record: default is to - replace extant bib with the imported record. + --no-replace overlay action for record: default is to + replace extant with the imported record. --comment optional comment to describe the record batch; if the comment has spaces in it, surround the --- a/svc/import_bib +++ a/svc/import_bib @@ -92,7 +92,7 @@ sub import_bib { $marcxml =~ s/<\?xml.*?\?>//i; # XXX we are ignoring the result of this; - BatchCommitBibRecords($batch_id, $framework) if lc($import_mode) eq 'direct'; + BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct'; $result->{'status'} = "ok"; $result->{'import_batch_id'} = $batch_id; --- a/tools/manage-marc-import.pl +++ a/tools/manage-marc-import.pl @@ -244,7 +244,7 @@ sub commit_batch { $callback = progress_callback($job, $dbh); } my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = - BatchCommitBibRecords($import_batch_id, $framework, 50, $callback); + BatchCommitRecords($import_batch_id, $framework, 50, $callback); $dbh->commit(); my $results = { @@ -273,7 +273,7 @@ sub revert_batch { $callback = progress_callback($job, $dbh); } my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = - BatchRevertBibRecords($import_batch_id, 50, $callback); + BatchRevertRecords($import_batch_id, 50, $callback); $dbh->commit(); my $results = { --