From 478ae788e7c1da7d8378a4504b1d61108a23f700 Mon Sep 17 00:00:00 2001 From: Jared Camins-Esakov Date: Mon, 21 May 2012 13:12:12 -0400 Subject: [PATCH] Bug 7475: Enable authority overlay Content-Type: text/plain; charset="UTF-8" This patch does the following: * Completes authority support within Koha's import batch code * Completes authority support in Koha's matching rule code * Exposes the ability to stage and commit batches of authorities using the renamed scripts misc/stage_file.pl and misc/commit_file.pl To test (note that these instructions assume you have a MARC21 installation and are using the provided sample file): 1. Find a file of authorities (a sample file with MARC21 authorities is attached to bug 7475) and download it to your server 2. Stage the file using the following command (replace with the name of the file you saved in step 1): > misc/stage_file.pl --file --authorities 3. Note the batch number the script assigns to your batch 4. Commit the records using the following command (replace with the batch number you made note of in step 3): > misc/commit_file.pl --batch-number 5. Index the authorities Zebraqueue (or wait) 6. Confirm that the new authorities appear. 7. Create a matching rule with the following settings: Code: AUTHTEST Description: Personal name main entry Match threshold: 999 Record type: Authority record Search index: Heading-main Score: 1000 Tag: 100 Subfields: a Offset: 0 Length: 0 (note the ID of this matching rule) 8. Stage the authority file again, this time using the following command: > misc/stage_file.pl --file --authorities \ --match 7. Revert the import with the following command: > misc/commit_file.pl --batch-number --revert 8. Index the authorities Zebraqueue (or wait) 9. Confirm that the records have been removed --- C4/ImportBatch.pm | 320 +++++++++++++------ C4/Matcher.pm | 73 ++++- admin/matching-rules.pl | 12 +- .../data/mysql/atomicupdate/importauthorities.pl | 19 ++ installer/data/mysql/kohastructure.sql | 15 + .../prog/en/modules/admin/matching-rules.tt | 11 + misc/{commit_biblios_file.pl => commit_file.pl} | 54 +++- misc/cronjobs/import_webservice_batch.pl | 2 +- misc/{stage_biblios_file.pl => stage_file.pl} | 84 +++--- svc/import_bib | 2 +- tools/manage-marc-import.pl | 4 +- 11 files changed, 425 insertions(+), 171 deletions(-) create mode 100755 installer/data/mysql/atomicupdate/importauthorities.pl rename misc/{commit_biblios_file.pl => commit_file.pl} (63%) rename misc/{stage_biblios_file.pl => stage_file.pl} (67%) diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index a64a0d2..007f339 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -1,6 +1,6 @@ package C4::ImportBatch; -# Copyright (C) 2007 LibLime +# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services # # This file is part of Koha. # @@ -40,14 +40,16 @@ BEGIN { GetImportRecordMarcXML AddImportBatch GetImportBatch + AddAuthorityToBatch AddBiblioToBatch AddItemsToImportBiblio + ModAuthorityInBatch ModBiblioInBatch BatchStageMarcRecords - BatchFindBibDuplicates - BatchCommitBibRecords - BatchRevertBibRecords + BatchFindDuplicates + BatchCommitRecords + BatchRevertRecords CleanBatch GetAllImportBatches @@ -272,10 +274,44 @@ sub ModBiblioInBatch { } +=head2 AddAuthToBatch + + my $import_record_id = AddAuthToBatch($batch_id, $record_sequence, + $marc_record, $encoding, $z3950random, $update_counts); + +=cut + +sub AddAuthToBatch { + my $batch_id = shift; + my $record_sequence = shift; + my $marc_record = shift; + my $encoding = shift; + my $z3950random = shift; + my $update_counts = @_ ? shift : 1; + + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random); + _add_auth_fields($import_record_id, $marc_record); + _update_batch_record_counts($batch_id) if $update_counts; + return $import_record_id; +} + +=head2 ModAuthInBatch + + ModAuthInBatch($import_record_id, $marc_record); + +=cut + +sub ModAuthInBatch { + my ($import_record_id, $marc_record) = @_; + + _update_import_record_marc($import_record_id, $marc_record); + +} + =head2 BatchStageMarcRecords ($batch_id, $num_records, $num_items, @invalid_records) = - BatchStageMarcRecords($encoding, $marc_records, $file_name, + BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name, $comments, $branch_code, $parse_items, $leave_as_staging, $progress_interval, $progress_callback); @@ -283,6 +319,7 @@ sub ModBiblioInBatch { =cut sub BatchStageMarcRecords { + my $record_type = shift; my $encoding = shift; my $marc_records = shift; my $file_name = shift; @@ -338,10 +375,14 @@ sub BatchStageMarcRecords { push @invalid_records, $marc_blob; } else { $num_valid++; - $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); - if ($parse_items) { - my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); - $num_items += scalar(@import_items_ids); + if ($record_type eq 'biblio') { + $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); + if ($parse_items) { + my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); + $num_items += scalar(@import_items_ids); + } + } elsif ($record_type eq 'auth') { + $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); } } } @@ -392,9 +433,9 @@ sub AddItemsToImportBiblio { return @import_items_ids; } -=head2 BatchFindBibDuplicates +=head2 BatchFindDuplicates - my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, + my $num_with_matches = BatchFindDuplicates($batch_id, $matcher, $max_matches, $progress_interval, $progress_callback); Goes through the records loaded in the batch and attempts to @@ -412,7 +453,7 @@ singular argument. =cut -sub BatchFindBibDuplicates { +sub BatchFindDuplicates { my $batch_id = shift; my $matcher = shift; my $max_matches = @_ ? shift : 10; @@ -430,9 +471,10 @@ sub BatchFindBibDuplicates { my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, marc + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, marc FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); my $num_with_matches = 0; @@ -460,15 +502,15 @@ sub BatchFindBibDuplicates { return $num_with_matches; } -=head2 BatchCommitBibRecords +=head2 BatchCommitRecords - my ($num_added, $num_updated, $num_items_added, $num_items_errored, - $num_ignored) = BatchCommitBibRecords($batch_id, $framework, - $progress_interval, $progress_callback); + my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + BatchCommitRecords($batch_id, $framework, + $progress_interval, $progress_callback); =cut -sub BatchCommitBibRecords { +sub BatchCommitRecords { my $batch_id = shift; my $framework = shift; @@ -483,25 +525,29 @@ sub BatchCommitBibRecords { $progress_interval = 0 unless 'CODE' eq ref $progress_callback; } + my $record_type; my $num_added = 0; my $num_updated = 0; my $num_items_added = 0; my $num_items_errored = 0; my $num_ignored = 0; # commit (i.e., save, all records in the batch) - # FIXME biblio only at the moment SetImportBatchStatus('importing'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $item_action = GetImportBatchItemAction($batch_id); + my $item_tag; + my $item_subfield; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); my $rec_num = 0; while (my $rowref = $sth->fetchrow_hashref) { + $record_type = $rowref->{'record_type'}; $rec_num++; if ($progress_interval and (0 == ($rec_num % $progress_interval))) { &$progress_callback($rec_num); @@ -513,67 +559,87 @@ sub BatchCommitBibRecords { my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'}); - # remove any item tags - rely on BatchCommitItems - my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); - foreach my $item_field ($marc_record->field($item_tag)) { - $marc_record->delete_field($item_field); + if ($record_type eq 'biblio') { + # remove any item tags - rely on BatchCommitItems + ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); + foreach my $item_field ($marc_record->field($item_tag)) { + $marc_record->delete_field($item_field); + } } - # decide what what to do with the bib and item records - my ($bib_result, $item_result, $bib_match) = + my ($record_result, $item_result, $record_match) = _get_commit_action($overlay_action, $nomatch_action, $item_action, - $rowref->{'overlay_status'}, $rowref->{'import_record_id'}); + $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type); - if ($bib_result eq 'create_new') { + my $recordid; + my $query; + if ($record_result eq 'create_new') { $num_added++; - my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, $framework); - my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth->execute($biblionumber, $rowref->{'import_record_id'}); - $sth->finish(); - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($record_type eq 'biblio') { + my $biblioitemnumber; + ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } + } else { + my $authid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record)); + $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; } + my $sth = $dbh->prepare_cached($query); + $sth->execute($recordid, $rowref->{'import_record_id'}); + $sth->finish(); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } elsif ($bib_result eq 'replace') { + } elsif ($record_result eq 'replace') { $num_updated++; - my $biblionumber = $bib_match; - my ($count, $oldbiblio) = GetBiblio($biblionumber); - my $oldxml = GetXmlBiblio($biblionumber); - - # remove item fields so that they don't get - # added again if record is reverted - my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}); - foreach my $item_field ($old_marc->field($item_tag)) { - $old_marc->delete_field($item_field); - } + $recordid = $record_match; + my $oldxml; + if ($record_type eq 'biblio') { + my ($count, $oldbiblio) = GetBiblio($recordid); + $oldxml = GetXmlBiblio($recordid); + + # remove item fields so that they don't get + # added again if record is reverted + my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}); + foreach my $item_field ($old_marc->field($item_tag)) { + $old_marc->delete_field($item_field); + } + $oldxml = $old_marc->as_xml(); + + ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'}); + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } + } else { + my $oldxml = GetAuthorityXML($recordid); - ModBiblio($marc_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record)); + $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; + } my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?"); - $sth->execute($old_marc->as_xml(), $rowref->{'import_record_id'}); + $sth->execute($oldxml, $rowref->{'import_record_id'}); $sth->finish(); - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + my $sth2 = $dbh->prepare_cached($query); + $sth2->execute($recordid, $rowref->{'import_record_id'}); $sth2->finish(); - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; - } SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } elsif ($bib_result eq 'ignore') { + } elsif ($record_result eq 'ignore') { $num_ignored++; - my $biblionumber = $bib_match; - if (defined $biblionumber and $item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); $num_items_added += $bib_items_added; $num_items_errored += $bib_items_errored; # still need to record the matched biblionumber so that the # items can be reverted my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + $sth2->execute($recordid, $rowref->{'import_record_id'}); SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); } SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); @@ -632,63 +698,84 @@ sub BatchCommitItems { return ($num_items_added, $num_items_errored); } -=head2 BatchRevertBibRecords +=head2 BatchRevertRecords my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, - $num_ignored) = BatchRevertBibRecords($batch_id); + $num_ignored) = BatchRevertRecords($batch_id); =cut -sub BatchRevertBibRecords { +sub BatchRevertRecords { my $batch_id = shift; + my $record_type; my $num_deleted = 0; my $num_errors = 0; my $num_reverted = 0; - my $num_items_deleted = 0; my $num_ignored = 0; + my $num_items_deleted = 0; # commit (i.e., save, all records in the batch) - # FIXME biblio only at the moment SetImportBatchStatus('reverting'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); while (my $rowref = $sth->fetchrow_hashref) { + $record_type = $rowref->{'record_type'}; if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') { $num_ignored++; next; } - my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); + my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); - if ($bib_result eq 'delete') { - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - my $error = DelBiblio($rowref->{'matched_biblionumber'}); + if ($record_result eq 'delete') { + my $error = undef; + if ($record_type eq 'biblio') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + $error = DelBiblio($rowref->{'matched_biblionumber'}); + } else { + my $deletedauthid = DelAuthority($rowref->{'matched_authid'}); + } if (defined $error) { $num_errors++; } else { $num_deleted++; SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - } elsif ($bib_result eq 'restore') { + } elsif ($record_result eq 'restore') { $num_reverted++; my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}); - my $biblionumber = $rowref->{'matched_biblionumber'}; - my ($count, $oldbiblio) = GetBiblio($biblionumber); - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + if ($record_type eq 'biblio') { + my $biblionumber = $rowref->{'matched_biblionumber'}; + my ($count, $oldbiblio) = GetBiblio($biblionumber); + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + } else { + my $authid = $rowref->{'matched_authid'}; + ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record)); + } SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); - } elsif ($bib_result eq 'ignore') { - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + } elsif ($record_result eq 'ignore') { + if ($record_type eq 'biblio') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + } SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - # remove matched_biblionumber only if there is no 'imported' item left - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')" ); + my $query; + if ($record_type eq 'biblio') { + # remove matched_biblionumber only if there is no 'imported' item left + $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; + $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')"; + } else { + $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?"; + } + my $sth2 = $dbh->prepare_cached($query); $sth2->execute($rowref->{'import_record_id'}); } @@ -1263,6 +1350,18 @@ sub _add_biblio_fields { } +sub _add_auth_fields { + my ($import_record_id, $marc_record) = @_; + + my $dbh = C4::Context->dbh; + # FIXME no controlnumber + my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id) VALUES (?)"); + $sth->execute($import_record_id); + $sth->finish(); + +} + + sub _update_biblio_fields { my ($import_record_id, $marc_record) = @_; @@ -1311,26 +1410,45 @@ sub _update_batch_record_counts { } sub _get_commit_action { - my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_; + my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_; - my ($bib_result, $bib_match, $item_result); - - if ($overlay_status ne 'no_match') { - $bib_match = GetBestRecordMatch($import_record_id); - if ($overlay_action eq 'replace') { - $bib_result = defined($bib_match) ? 'replace' : 'create_new'; - } elsif ($overlay_action eq 'create_new') { - $bib_result = 'create_new'; - } elsif ($overlay_action eq 'ignore') { - $bib_result = 'ignore'; - } - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; - } else { - $bib_result = $nomatch_action; - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; - } + if ($record_type eq 'biblio') { + my ($bib_result, $bib_match, $item_result); + + if ($overlay_status ne 'no_match') { + $bib_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $bib_result = defined($bib_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $bib_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $bib_result = 'ignore'; + } + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; + } else { + $bib_result = $nomatch_action; + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; + } + return ($bib_result, $item_result, $bib_match); + } else { # must be auths + my ($auth_result, $auth_match); + + if ($overlay_status ne 'no_match') { + $auth_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $auth_result = defined($auth_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $auth_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $auth_result = 'ignore'; + } + } else { + $auth_result = $nomatch_action; + } - return ($bib_result, $item_result, $bib_match); + return ($auth_result, undef, $auth_match); + + } } sub _get_revert_action { diff --git a/C4/Matcher.pm b/C4/Matcher.pm index 59b5876..c9f418d 100644 --- a/C4/Matcher.pm +++ b/C4/Matcher.pm @@ -1,6 +1,6 @@ package C4::Matcher; -# Copyright (C) 2007 LibLime +# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services # # This file is part of Koha. # @@ -22,8 +22,6 @@ use warnings; use C4::Context; use MARC::Record; -use C4::Search; -use C4::Biblio; use vars qw($VERSION); @@ -384,6 +382,20 @@ sub delete { $sth->execute($matcher_id); # relying on cascading deletes to clean up everything } +=head2 record_type + + $matcher->record_type('biblio'); + my $record_type = $matcher->record_type(); + +Accessor method. + +=cut + +sub record_type { + my $self = shift; + @_ ? $self->{'record_type'} = shift : $self->{'record_type'}; +} + =head2 threshold $matcher->threshold(1000); @@ -582,7 +594,7 @@ sub add_simple_required_check { ); } -=head2 find_matches +=head2 get_matches my @matches = $matcher->get_matches($marc_record, $max_matches); foreach $match (@matches) { @@ -618,9 +630,34 @@ sub get_matches { my @source_keys = _get_match_keys($source_record, $matchpoint); next if scalar(@source_keys) == 0; # build query - my $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys); - # FIXME only searching biblio index at the moment - my ($error, $searchresults, $total_hits) = SimpleSearch($query, 0, $max_matches); + my $query; + my $error; + my $searchresults; + my $total_hits; + if ($self->{'record_type'} eq 'biblio') { + $query = join(" or ", map { "$matchpoint->{'index'}=$_" } @source_keys); +# FIXME only searching biblio index at the moment + require C4::Search; + ($error, $searchresults, $total_hits) = C4::Search::SimpleSearch($query, 0, $max_matches); + } elsif ($self->{'record_type'} eq 'authority') { + my $authresults; + my @marclist; + my @and_or; + my @excluding = []; + my @operator; + my @value; + foreach my $key (@source_keys) { + push @marclist, $matchpoint->{'index'}; + push @and_or, 'or'; + push @operator, 'exact'; + push @value, $key; + } + require C4::AuthoritiesMarc; + ($searchresults, $total_hits) = C4::AuthoritiesMarc::SearchAuthorities( + \@marclist, \@and_or, \@excluding, \@operator, + \@value, 0, 20, undef, 'AuthidAsc', 1 + ); + } if (defined $error ) { warn "search failed ($query) $error"; @@ -639,13 +676,20 @@ sub get_matches { keys %matches; my @results = (); - foreach my $marcblob (keys %matches) { - my $target_record = MARC::Record->new_from_usmarc($marcblob); - my $result = TransformMarcToKoha(C4::Context->dbh, $target_record, ''); - # FIXME - again, bibliospecific - # also, can search engine be induced to give just the number in the first place? - my $record_number = $result->{'biblionumber'}; - push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} }; + if ($self->{'record_type'} eq 'biblio') { + require C4::Biblio; + foreach my $marcblob (keys %matches) { + my $target_record = MARC::Record->new_from_usmarc($marcblob); + my $record_number; + my $result = C4::Biblio::TransformMarcToKoha(C4::Context->dbh, $target_record, ''); + $record_number = $result->{'biblionumber'}; + push @results, { 'record_id' => $record_number, 'score' => $matches{$marcblob} }; + } + } elsif ($self->{'record_type'} eq 'authority') { + require C4::AuthoritiesMarc; + foreach my $authid (keys %matches) { + push @results, { 'record_id' => $authid, 'score' => $matches{$authid} }; + } } @results = sort { $b->{'score'} cmp $a->{'score'} } @results; if (scalar(@results) > $max_matches) { @@ -673,6 +717,7 @@ sub dump { $result->{'matcher_id'} = $self->{'id'}; $result->{'code'} = $self->{'code'}; $result->{'description'} = $self->{'description'}; + $result->{'record_type'} = $self->{'record_type'}; $result->{'matchpoints'} = []; foreach my $matchpoint (@{ $self->{'matchpoints'} }) { diff --git a/admin/matching-rules.pl b/admin/matching-rules.pl index b167026..e682fb5 100755 --- a/admin/matching-rules.pl +++ b/admin/matching-rules.pl @@ -92,9 +92,10 @@ sub add_matching_rule_form { sub add_update_matching_rule { my $template = shift; my $matcher_id = shift; + my $record_type = $input->param('record_type') || 'biblio'; # do parsing - my $matcher = C4::Matcher->new('biblio', 1000); # FIXME biblio only for now + my $matcher = C4::Matcher->new($record_type, 1000); $matcher->code($input->param('code')); $matcher->description($input->param('description')); $matcher->threshold($input->param('threshold')); @@ -203,10 +204,11 @@ sub edit_matching_rule_form { my $matcher = C4::Matcher->fetch($matcher_id); - $template->param(matcher_id => $matcher_id); - $template->param(code => $matcher->code()); - $template->param(description => $matcher->description()); - $template->param(threshold => $matcher->threshold()); + $template->{VARS}->{'matcher_id'} = $matcher_id; + $template->{VARS}->{'code'} = $matcher->code(); + $template->{VARS}->{'description'} = $matcher->description(); + $template->{VARS}->{'threshold'} = $matcher->threshold(); + $template->{VARS}->{'record_type'} = $matcher->record_type(); my $matcher_info = $matcher->dump(); my @matchpoints = (); diff --git a/installer/data/mysql/atomicupdate/importauthorities.pl b/installer/data/mysql/atomicupdate/importauthorities.pl new file mode 100755 index 0000000..f242f61 --- /dev/null +++ b/installer/data/mysql/atomicupdate/importauthorities.pl @@ -0,0 +1,19 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use C4::Context; +my $dbh = C4::Context->dbh; + +$dbh->do( +q|CREATE TABLE `import_auths` ( + `import_record_id` int(11) NOT NULL, + `matched_authid` int(11) default NULL, + `control_number` varchar(25) default NULL, + `original_source` varchar(25) default NULL, + CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`) + REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE, + KEY `matched_authid` (`matched_authid`) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;| +); +print "Upgrade done (Added support for staging authorities)\n"; diff --git a/installer/data/mysql/kohastructure.sql b/installer/data/mysql/kohastructure.sql index 54498c3..13864ce 100644 --- a/installer/data/mysql/kohastructure.sql +++ b/installer/data/mysql/kohastructure.sql @@ -909,6 +909,21 @@ CREATE TABLE `import_record_matches` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- +-- Table structure for table `import_auths` +-- + +DROP TABLE IF EXISTS `import_auths`; +CREATE TABLE `import_auths` ( + `import_record_id` int(11) NOT NULL, + `matched_authid` int(11) default NULL, + `control_number` varchar(25) default NULL, + `original_source` varchar(25) default NULL, + CONSTRAINT `import_auths_ibfk_1` FOREIGN KEY (`import_record_id`) + REFERENCES `import_records` (`import_record_id`) ON DELETE CASCADE ON UPDATE CASCADE, + KEY `matched_authid` (`matched_authid`), +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- -- Table structure for table `import_biblios` -- diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt index 5835244..2bd8aaf 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/matching-rules.tt @@ -220,6 +220,17 @@ function CheckRuleForm(f) { Required +
  • + Required +
    • diff --git a/misc/commit_biblios_file.pl b/misc/commit_file.pl similarity index 63% rename from misc/commit_biblios_file.pl rename to misc/commit_file.pl index b4be670..1f54faf 100755 --- a/misc/commit_biblios_file.pl +++ b/misc/commit_file.pl @@ -18,11 +18,13 @@ $| = 1; # command-line parameters my $batch_number = ""; my $list_batches = 0; +my $revert = 0; my $want_help = 0; my $result = GetOptions( 'batch-number:s' => \$batch_number, 'list-batches' => \$list_batches, + 'revert' => \$revert, 'h|help' => \$want_help ); @@ -45,9 +47,15 @@ $dbh->{AutoCommit} = 0; if ($batch_number =~ /^\d+$/ and $batch_number > 0) { my $batch = GetImportBatch($batch_number); die "$0: import batch $batch_number does not exist in database\n" unless defined $batch; - die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" - unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted"; - process_batch($batch_number); + if ($revert) { + die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" + unless $batch->{'import_status'} eq "imported"; + revert_batch($batch_number); + } else { + die "$0: import batch $batch_number status is '" . $batch->{'import_status'} . "', and therefore cannot be imported\n" + unless $batch->{'import_status'} eq "staged" or $batch->{'import_status'} eq "reverted"; + process_batch($batch_number); + } $dbh->commit(); } else { die "$0: please specify a numeric batch ID\n"; @@ -74,8 +82,8 @@ sub process_batch { my ($import_batch_id) = @_; print "... importing MARC records -- please wait\n"; - my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = - BatchCommitBibRecords($import_batch_id, '', 100, \&print_progress_and_commit); + my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + BatchCommitRecords($import_batch_id, '', 100, \&print_progress_and_commit); print "... finished importing MARC records\n"; print <<_SUMMARY_; @@ -83,17 +91,40 @@ sub process_batch { MARC record import report ---------------------------------------- Batch number: $import_batch_id -Number of new bibs added: $num_added -Number of bibs replaced: $num_updated -Number of bibs ignored: $num_ignored +Number of new records added: $num_added +Number of records replaced: $num_updated +Number of records ignored: $num_ignored Number of items added: $num_items_added Number of items ignored: $num_items_errored -Note: an item is ignored if its barcode is a +Note: an item is ignored if its barcode is a duplicate of one already in the database. _SUMMARY_ } +sub revert_batch { + my ($import_batch_id) = @_; + + print "... reverting batch -- please wait\n"; + my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = + BatchRevertRecords($import_batch_id, 100, \&print_progress_and_commit); + print "... finished reverting batch\n"; + + print <<_SUMMARY_; + +MARC record import report +---------------------------------------- +Batch number: $import_batch_id +Number of records deleted: $num_deleted +Number of errors: $num_errors +Number of records reverted: $num_reverted +Number of records ignored: $num_ignored +Number of items added: $num_items_deleted + +_SUMMARY_ +} + + sub print_progress_and_commit { my $recs = shift; print "... processed $recs records\n"; @@ -106,7 +137,7 @@ $0: import a batch of staged MARC records into database. Use this batch job to complete the import of a batch of MARC records that was staged either by the batch job -stage_biblios_file.pl or by the Koha Tools option +stage_file.pl or by the Koha Tools option "Stage MARC Records for Import". Parameters: @@ -114,6 +145,7 @@ Parameters: to import --list-batches print a list of record batches available to commit - --help or -h show this message. + --revert revert a batch instead of importing it + --help or -h show this message. _USAGE_ } diff --git a/misc/cronjobs/import_webservice_batch.pl b/misc/cronjobs/import_webservice_batch.pl index d1b30ef..3f57779 100755 --- a/misc/cronjobs/import_webservice_batch.pl +++ b/misc/cronjobs/import_webservice_batch.pl @@ -54,4 +54,4 @@ EOF my $batch_ids = GetStagedWebserviceBatches() or exit; $framework ||= ''; -BatchCommitBibRecords($_, $framework) foreach @$batch_ids; +BatchCommitRecords($_, $framework) foreach @$batch_ids; diff --git a/misc/stage_biblios_file.pl b/misc/stage_file.pl similarity index 67% rename from misc/stage_biblios_file.pl rename to misc/stage_file.pl index 6e2b30c..4ce16b5 100755 --- a/misc/stage_biblios_file.pl +++ b/misc/stage_file.pl @@ -4,6 +4,7 @@ # # Copyright (C) 2007 LibLime # Parts Copyright BSZ 2011 +# Parts Copyright C & P Bibliography Services 2012 # # Koha is free software; you can redistribute it and/or modify it under the # terms of the GNU General Public License as published by the Free Software @@ -35,8 +36,10 @@ use Getopt::Long; $| = 1; # command-line parameters +my $record_type = "biblio"; my $encoding = ""; -my $match_bibs = 0; +my $authorities = 0; +my $match = 0; my $add_items = 0; my $input_file = ""; my $batch_comment = ""; @@ -46,13 +49,16 @@ my $no_replace ; my $result = GetOptions( 'encoding:s' => \$encoding, 'file:s' => \$input_file, - 'match-bibs:s' => \$match_bibs, + 'match|match-bibs:s' => \$match, 'add-items' => \$add_items, 'no-replace' => \$no_replace, 'comment:s' => \$batch_comment, + 'authorities' => \$authorities, 'h|help' => \$want_help ); +$record_type = 'auth' if ($authorities); + if ($encoding eq "") { $encoding = "utf8"; } @@ -68,13 +74,13 @@ unless (-r $input_file) { my $dbh = C4::Context->dbh; $dbh->{AutoCommit} = 0; -process_batch($input_file, $match_bibs, $add_items, $batch_comment); +process_batch($input_file, $record_type, $match, $add_items, $batch_comment); $dbh->commit(); exit 0; sub process_batch { - my ($input_file, $match_bibs, $add_items, $batch_comment) = @_; + my ($input_file, $record_type, $match, $add_items, $batch_comment) = @_; open IN, "<$input_file" or die "$0: cannot open input file $input_file: $!\n"; my $marc_records = ""; @@ -92,50 +98,53 @@ sub process_batch { close IN; print "... staging MARC records -- please wait\n"; - my ($batch_id, $num_valid, $num_items, @import_errors) = - BatchStageMarcRecords($encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0, + my ($batch_id, $num_valid_records, $num_items, @import_errors) = + BatchStageMarcRecords($record_type, $encoding, $marc_records, $input_file, $batch_comment, '', $add_items, 0, 100, \&print_progress_and_commit); print "... finished staging MARC records\n"; my $num_with_matches = 0; - if ($match_bibs) { - my $matcher = C4::Matcher->fetch($match_bibs) ; - if (! defined $matcher) { - $matcher = C4::Matcher->new('biblio'); + if ($match) { + my $matcher = C4::Matcher->fetch($match) ; + if (defined $matcher) { + SetImportBatchMatcher($batch_id, $match); + } elsif ($record_type eq 'biblio') { + $matcher = C4::Matcher->new($record_type); $matcher->add_simple_matchpoint('isbn', 1000, '020', 'a', -1, 0, ''); - $matcher->add_simple_required_check('245', 'a', -1, 0, '', + $matcher->add_simple_required_check('245', 'a', -1, 0, '', '245', 'a', -1, 0, ''); - } else { - SetImportBatchMatcher($batch_id, $match_bibs); } # set default record overlay behavior SetImportBatchOverlayAction($batch_id, ($no_replace) ? 'ignore' : 'replace'); SetImportBatchNoMatchAction($batch_id, 'create_new'); SetImportBatchItemAction($batch_id, 'always_add'); print "... looking for matches with records already in database\n"; - $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); + $num_with_matches = BatchFindDuplicates($batch_id, $matcher, 10, 100, \&print_progress_and_commit); print "... finished looking for matches\n"; } - my $num_invalid_bibs = scalar(@import_errors); + my $num_invalid_records = scalar(@import_errors); print <<_SUMMARY_; MARC record staging report ------------------------------------ -Input file: $input_file -Number of input bibs: $num_input_records -Number of valid bibs: $num_valid -Number of invalid bibs: $num_invalid_bibs +Input file: $input_file +Record type: $record_type +Number of input records: $num_input_records +Number of valid records: $num_valid_records +Number of invalid records: $num_invalid_records _SUMMARY_ - if ($match_bibs) { - print "Number of bibs matched: $num_with_matches\n"; + if ($match) { + print "Number of records matched: $num_with_matches\n"; } else { - print "Incoming bibs not matched against existing bibs (--match-bibs option not supplied)\n"; + print "Incoming records not matched against existing records (--match option not supplied)\n"; } - if ($add_items) { - print "Number of items parsed: $num_items\n"; - } else { - print "No items parsed (--add-items option not supplied)\n"; + if ($record_type eq 'biblio') { + if ($add_items) { + print "Number of items parsed: $num_items\n"; + } else { + print "No items parsed (--add-items option not supplied)\n"; + } } print "\n"; @@ -151,31 +160,34 @@ sub print_progress_and_commit { sub print_usage { print <<_USAGE_; -$0: stage MARC bib file into reservoir. +$0: stage MARC file into reservoir. -Use this batch job to load a file of MARC bibliographic records -(with optional item information) into the Koha reservoir. +Use this batch job to load a file of MARC bibliographic +(with optional item information) or authority records into +the Koha reservoir. After running this program to stage your file, you can use -either the batch job commit_biblios_file.pl or the Koha +either the batch job commit_file.pl or the Koha Tools option "Manage Staged MARC Records" to load the records into the main Koha database. Parameters: --file name of input MARC bib file + --authorities stage authority records instead of bibs --encoding encoding of MARC records, default is utf8. Other possible options are: MARC-8, ISO_5426, ISO_6937, ISO_8859-1, EUC-KR - --match-bibs use this option to match bibs - in the file with bibs already in + --match use this option to match records + in the file with records already in the database for future overlay. - If isn't defined, a default - MARC21 ISBN & title match rule will be applied. + If isn't defined, a default + MARC21 ISBN & title match rule will be applied + for bib imports. --add-items use this option to specify that item data is embedded in the MARC bibs and should be parsed. - --no-replace overlay action for bib record: default is to - replace extant bib with the imported record. + --no-replace overlay action for record: default is to + replace extant with the imported record. --comment optional comment to describe the record batch; if the comment has spaces in it, surround the diff --git a/svc/import_bib b/svc/import_bib index 5e7e493..c130204 100755 --- a/svc/import_bib +++ b/svc/import_bib @@ -92,7 +92,7 @@ sub import_bib { $marcxml =~ s/<\?xml.*?\?>//i; # XXX we are ignoring the result of this; - BatchCommitBibRecords($batch_id, $framework) if lc($import_mode) eq 'direct'; + BatchCommitRecords($batch_id, $framework) if lc($import_mode) eq 'direct'; $result->{'status'} = "ok"; $result->{'import_batch_id'} = $batch_id; diff --git a/tools/manage-marc-import.pl b/tools/manage-marc-import.pl index 8a8d9e8..7346c59 100755 --- a/tools/manage-marc-import.pl +++ b/tools/manage-marc-import.pl @@ -244,7 +244,7 @@ sub commit_batch { $callback = progress_callback($job, $dbh); } my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = - BatchCommitBibRecords($import_batch_id, $framework, 50, $callback); + BatchCommitRecords($import_batch_id, $framework, 50, $callback); $dbh->commit(); my $results = { @@ -273,7 +273,7 @@ sub revert_batch { $callback = progress_callback($job, $dbh); } my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = - BatchRevertBibRecords($import_batch_id, 50, $callback); + BatchRevertRecords($import_batch_id, 50, $callback); $dbh->commit(); my $results = { -- 1.7.2.5