Bugzilla – Attachment 2565 Details for
Bug 5166
Reintroduce a zebraqueue daemon, make several rebuild_zebra.pl functions a lib
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Initial patch
0002-Refactor-rebuild_zebra.pl-into-a-library.patch (text/plain), 16.36 KB, created by
Tomás Cohen Arazi (tcohen)
on 2010-08-24 16:23:37 UTC
(
hide
)
Description:
Initial patch
Filename:
MIME Type:
Creator:
Tomás Cohen Arazi (tcohen)
Created:
2010-08-24 16:23:37 UTC
Size:
16.36 KB
patch
obsolete
>From 81f726e2603aefc41fbba9a1398f3f5d87329d70 Mon Sep 17 00:00:00 2001 >From: =?UTF-8?q?Tom=C3=A1s=20Cohen=20Arazi?= <tomascohen@gmail.com> >Date: Tue, 24 Aug 2010 13:09:50 -0300 >Subject: [PATCH 2/2] [Bug #5166] Refactor rebuild_zebra.pl into a library > >--- > C4/Catalog/Rebuild.pm | 426 +++++++++++++++++++++++++++++++++++++++++++++++++ > 1 files changed, 426 insertions(+), 0 deletions(-) > create mode 100755 C4/Catalog/Rebuild.pm > >diff --git a/C4/Catalog/Rebuild.pm b/C4/Catalog/Rebuild.pm >new file mode 100755 >index 0000000..c022989 >--- /dev/null >+++ b/C4/Catalog/Rebuild.pm >@@ -0,0 +1,426 @@ >+package C4::Catalog::Rebuild; >+ >+use strict; >+use C4::Context; >+use Getopt::Long; >+use File::Temp qw/ tempdir /; >+use File::Path; >+use C4::Biblio; >+use C4::AuthoritiesMarc; >+ >+# library for checking for updates in zerbaqueue >+# can also check zebradir structure & create directories & >+# mandatory files if needed >+ >+ my $nosanitize = ''; >+ my $skip_export = ''; >+ my $keep_export = ''; >+ my $noxml = ''; >+ my $as_xml = ''; >+ my $process_zebraqueue = 1; >+ my $do_not_clear_zebraqueue = ''; >+ my $verbose_logging = ''; >+ my $reset = ''; >+ my $noshadow = ''; >+ my $biblioserverdir = C4::Context->zebraconfig('biblioserver')->{directory}; >+ my $authorityserverdir = C4::Context->zebraconfig('authorityserver')->{directory}; >+ my $kohadir = C4::Context->config('intranetdir'); >+ my $dbh = C4::Context->dbh; >+ my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber",""); >+ my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber",""); >+ >+ >+sub update_auth { >+ >+ # Create tmp dir >+ my $directory = File::Temp->newdir(); >+ my $zebraidx_log_opt = " -v none,fatal,warn "; >+ >+ >+ # Update authorities >+ index_records('authority', $directory, $skip_export, "1", $as_xml,\ >+ $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt); >+ >+ return 0; >+ >+} >+ >+sub update_auth_and_biblio { >+ >+ # Create tmp dir >+ my $directory = File::Temp->newdir(); >+ my $zebraidx_log_opt = " -v none,fatal,warn "; >+ >+ # Update authorities >+ index_records('authority', $directory, 0, 1, 1, 0, 0, 0, $verbose_logging, $zebraidx_log_opt); >+ >+ # Update biblios >+ index_records('biblio', $directory, 0, 1, 1, 0, 0, 0, $verbose_logging, $zebraidx_log_opt); >+ >+ return 0; >+} >+ >+sub index_records { >+ my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt) = @_; >+ >+ >+ my $num_records_exported = 0; >+ my $num_records_deleted = 0; >+ >+ if ($skip_export && $verbose_logging) { >+ print "====================\n"; >+ print "SKIPPING $record_type export\n"; >+ print "====================\n"; >+ } else { >+ if ( $verbose_logging ) { >+ print "====================\n"; >+ print "exporting $record_type\n"; >+ print "====================\n"; >+ } >+ mkdir "$directory" unless (-d $directory); >+ mkdir "$directory/$record_type" unless (-d "$directory/$record_type"); >+ if ($process_zebraqueue) { >+ my $entries = select_zebraqueue_records($record_type, 'deleted'); >+ mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type"); >+ $num_records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml); >+ mark_zebraqueue_batch_done($entries); >+ $entries = select_zebraqueue_records($record_type, 'updated'); >+ mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type"); >+ $num_records_exported = export_marc_records_from_list($record_type, >+ $entries, "$directory/upd_$record_type", $as_xml, $noxml); >+ mark_zebraqueue_batch_done($entries); >+ } else { >+ my $sth = select_all_records($record_type); >+ $num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize); >+ unless ($do_not_clear_zebraqueue) { >+ mark_all_zebraqueue_done($record_type); >+ } >+ } >+ } >+ >+ # >+ # and reindexing everything >+ # >+ if ( $verbose_logging ) { >+ print "====================\n"; >+ print "REINDEXING zebra\n"; >+ print "====================\n"; >+ } >+ my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ; >+ if ($process_zebraqueue) { >+ do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) >+ if $num_records_deleted; >+ do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) >+ if $num_records_exported; >+ } else { >+ do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt, $zebraidx_log_opt) >+ if ($num_records_exported or $skip_export); >+ } >+} >+ >+sub select_zebraqueue_records { >+ my ($record_type, $update_type) = @_; >+ >+ my $server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver'; >+ my $op = ($update_type eq 'deleted') ? 'recordDelete' : 'specialUpdate'; >+ >+ my $sth = $dbh->prepare("SELECT id, biblio_auth_number >+ FROM zebraqueue >+ WHERE server = ? >+ AND operation = ? >+ AND done = 0 >+ ORDER BY id DESC"); >+ $sth->execute($server, $op); >+ my $entries = $sth->fetchall_arrayref({}); >+} >+ >+sub mark_all_zebraqueue_done { >+ my ($record_type) = @_; >+ >+ my $server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver'; >+ >+ my $sth = $dbh->prepare("UPDATE zebraqueue SET done = 1 >+ WHERE server = ? >+ AND done = 0"); >+ $sth->execute($server); >+ print "MARK\n"; >+} >+ >+sub mark_zebraqueue_batch_done { >+ my ($entries) = @_; >+ >+ $dbh->{AutoCommit} = 0; >+ my $sth = $dbh->prepare("UPDATE zebraqueue SET done = 1 WHERE id = ?"); >+ $dbh->commit(); >+ foreach my $id (map { $_->{id} } @$entries) { >+ $sth->execute($id); >+ } >+ $dbh->{AutoCommit} = 1; >+} >+ >+sub select_all_records { >+ my $record_type = shift; >+ return ($record_type eq 'biblio') ? select_all_biblios() : select_all_authorities(); >+} >+ >+sub select_all_authorities { >+ my $sth = $dbh->prepare("SELECT authid FROM auth_header"); >+ $sth->execute(); >+ return $sth; >+} >+ >+sub select_all_biblios { >+ my $sth = $dbh->prepare("SELECT biblionumber FROM biblioitems ORDER BY biblionumber"); >+ $sth->execute(); >+ return $sth; >+} >+ >+sub export_marc_records_from_sth { >+ my ($record_type, $sth, $directory, $as_xml, $noxml, $nosanitize) = @_; >+ >+ my $num_exported = 0; >+ open (OUT, ">:utf8 ", "$directory/exported_records") or die $!; >+ my $i = 0; >+ while (my ($record_number) = $sth->fetchrow_array) { >+ print "." if ( $verbose_logging ); >+ print "\r$i" unless ($i++ %100 or !$verbose_logging); >+ if ( $nosanitize ) { >+ my $marcxml = $record_type eq 'biblio' >+ ? GetXmlBiblio( $record_number ) >+ : GetAuthorityXML( $record_number ); >+ if ( $marcxml ) { >+ print OUT $marcxml if $marcxml; >+ $num_exported++; >+ } >+ next; >+ } >+ my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml); >+ if (defined $marc) { >+ # FIXME - when more than one record is exported and $as_xml is true, >+ # the output file is not valid XML - it's just multiple <record> elements >+ # strung together with no single root element. zebraidx doesn't seem >+ # to care, though, at least if you're using the GRS-1 filter. It does >+ # care if you're using the DOM filter, which requires valid XML file(s). >+ print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc(); >+ $num_exported++; >+ } >+ } >+ print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); >+ close OUT; >+ return $num_exported; >+} >+ >+sub export_marc_records_from_list { >+ my ($record_type, $entries, $directory, $as_xml, $noxml) = @_; >+ >+ my $num_exported = 0; >+ open (OUT, ">:utf8 ", "$directory/exported_records") or die $!; >+ my $i = 0; >+ my %found = (); >+ foreach my $record_number ( map { $_->{biblio_auth_number} } >+ grep { !$found{ $_->{biblio_auth_number} }++ } >+ @$entries ) { >+ print "." if ( $verbose_logging ); >+ print "\r$i" unless ($i++ %100 or !$verbose_logging); >+ my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml); >+ if (defined $marc) { >+ # FIXME - when more than one record is exported and $as_xml is true, >+ # the output file is not valid XML - it's just multiple <record> elements >+ # strung together with no single root element. zebraidx doesn't seem >+ # to care, though, at least if you're using the GRS-1 filter. It does >+ # care if you're using the DOM filter, which requires valid XML file(s). >+ print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc(); >+ $num_exported++; >+ } >+ } >+ print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); >+ close OUT; >+ return $num_exported; >+} >+ >+sub generate_deleted_marc_records { >+ my ($record_type, $entries, $directory, $as_xml) = @_; >+ >+ my $num_exported = 0; >+ open (OUT, ">:utf8 ", "$directory/exported_records") or die $!; >+ my $i = 0; >+ foreach my $record_number (map { $_->{biblio_auth_number} } @$entries ) { >+ print "\r$i" unless ($i++ %100 or !$verbose_logging); >+ print "." if ( $verbose_logging ); >+ >+ my $marc = MARC::Record->new(); >+ if ($record_type eq 'biblio') { >+ fix_biblio_ids($marc, $record_number, $record_number); >+ } else { >+ fix_authority_id($marc, $record_number); >+ } >+ if (C4::Context->preference("marcflavour") eq "UNIMARC") { >+ fix_unimarc_100($marc); >+ } >+ >+ print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc(); >+ $num_exported++; >+ } >+ print "\nRecords exported: $num_exported\n" if ( $verbose_logging ); >+ close OUT; >+ return $num_exported; >+ >+ >+} >+ >+sub get_corrected_marc_record { >+ my ($record_type, $record_number, $noxml) = @_; >+ >+ my $marc = get_raw_marc_record($record_type, $record_number, $noxml); >+ >+ if (defined $marc) { >+ fix_leader($marc); >+ if ($record_type eq 'biblio') { >+ my $succeeded = fix_biblio_ids($marc, $record_number); >+ return unless $succeeded; >+ } else { >+ fix_authority_id($marc, $record_number); >+ } >+ if (C4::Context->preference("marcflavour") eq "UNIMARC") { >+ fix_unimarc_100($marc); >+ } >+ } >+ >+ return $marc; >+} >+ >+sub get_raw_marc_record { >+ my ($record_type, $record_number, $noxml) = @_; >+ >+ my $marc; >+ if ($record_type eq 'biblio') { >+ if ($noxml) { >+ my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?"); >+ $fetch_sth->execute($record_number); >+ if (my ($blob) = $fetch_sth->fetchrow_array) { >+ $marc = MARC::Record->new_from_usmarc($blob); >+ $fetch_sth->finish(); >+ } else { >+ return; # failure to find a bib is not a problem - >+ # a delete could have been done before >+ # trying to process a record update >+ } >+ } else { >+ eval { $marc = GetMarcBiblio($record_number); }; >+ if ($@) { >+ # here we do warn since catching an exception >+ # means that the bib was found but failed >+ # to be parsed >+ warn "error retrieving biblio $record_number"; >+ return; >+ } >+ } >+ } else { >+ eval { $marc = GetAuthority($record_number); }; >+ if ($@) { >+ warn "error retrieving authority $record_number"; >+ return; >+ } >+ } >+ return $marc; >+} >+ >+sub fix_leader { >+ # FIXME - this routine is suspect >+ # It blanks the Leader/00-05 and Leader/12-16 to >+ # force them to be recalculated correct when >+ # the $marc->as_usmarc() or $marc->as_xml() is called. >+ # But why is this necessary? It would be a serious bug >+ # in MARC::Record (definitely) and MARC::File::XML (arguably) >+ # if they are emitting incorrect leader values. >+ my $marc = shift; >+ >+ my $leader = $marc->leader; >+ substr($leader, 0, 5) = ' '; >+ substr($leader, 10, 7) = '22 '; >+ $marc->leader(substr($leader, 0, 24)); >+} >+ >+sub fix_biblio_ids { >+ # FIXME - it is essential to ensure that the biblionumber is present, >+ # otherwise, Zebra will choke on the record. However, this >+ # logic belongs in the relevant C4::Biblio APIs. >+ my $marc = shift; >+ my $biblionumber = shift; >+ my $biblioitemnumber; >+ if (@_) { >+ $biblioitemnumber = shift; >+ } else { >+ my $sth = $dbh->prepare( >+ "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?"); >+ $sth->execute($biblionumber); >+ ($biblioitemnumber) = $sth->fetchrow_array; >+ $sth->finish; >+ unless ($biblioitemnumber) { >+ warn "failed to get biblioitemnumber for biblio $biblionumber"; >+ return 0; >+ } >+ } >+ >+ # FIXME - this is cheating on two levels >+ # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function >+ # 2. Making sure that the biblionumber and biblioitemnumber are correct and >+ # present in the MARC::Record object ought to be part of GetMarcBiblio. >+ # >+ # On the other hand, this better for now than what rebuild_zebra.pl used to >+ # do, which was duplicate the code for inserting the biblionumber >+ # and biblioitemnumber >+ C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber); >+ >+ return 1; >+} >+ >+sub fix_authority_id { >+ # FIXME - as with fix_biblio_ids, the authid must be present >+ # for Zebra's sake. However, this really belongs >+ # in C4::AuthoritiesMarc. >+ my ($marc, $authid) = @_; >+ unless ($marc->field('001') and $marc->field('001')->data() eq $authid){ >+ $marc->delete_field($marc->field('001')); >+ $marc->insert_fields_ordered(MARC::Field->new('001',$authid)); >+ } >+} >+ >+sub fix_unimarc_100 { >+ # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc. >+ my $marc = shift; >+ >+ my $string; >+ if ( length($marc->subfield( 100, "a" )) == 35 ) { >+ $string = $marc->subfield( 100, "a" ); >+ my $f100 = $marc->field(100); >+ $marc->delete_field($f100); >+ } >+ else { >+ $string = POSIX::strftime( "%Y%m%d", localtime ); >+ $string =~ s/\-//g; >+ $string = sprintf( "%-*s", 35, $string ); >+ } >+ substr( $string, 22, 6, "frey50" ); >+ unless ( length($marc->subfield( 100, "a" )) == 35 ) { >+ $marc->delete_field($marc->field(100)); >+ $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string )); >+ } >+} >+ >+sub do_indexing { >+ my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format, $zebraidx_log_opt) = @_; >+ >+ my $zebra_server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver'; >+ my $zebra_db_name = ($record_type eq 'biblio') ? 'biblios' : 'authorities'; >+ my $zebra_config = C4::Context->zebraconfig($zebra_server)->{'config'}; >+ my $zebra_db_dir = C4::Context->zebraconfig($zebra_server)->{'directory'}; >+ >+ system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name init") if $reset_index; >+ system("zebraidx -c $zebra_config $zebraidx_log_opt $noshadow -g $record_format -d $zebra_db_name $op $record_dir"); >+ system("zebraidx -c $zebra_config $zebraidx_log_opt -g $record_format -d $zebra_db_name commit") unless $noshadow; >+ >+} >+ >+ >+1; >-- >1.7.0.4 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 5166
:
2565
|
3239
|
3380
|
3381
|
3382
|
3383
|
3643
|
3664
|
3958
|
4031
|
4831
|
6902
|
8357