Attachment #162015 for bug #35345

View | Details | Raw Unified | Return to bug 35345
Collapse All | Expand All




use Koha::Database;
use Modern::Perl;


use base qw(Koha::MetadataRecord);

__PACKAGE__->mk_accessors(qw( record schema id datatype ));

=cut

sub new {
    my $class        = shift;
    my $record       = shift;
    my $biblionumber = shift;

    my $self = $class->SUPER::new(

=cut

sub get_from_biblionumber {
    my ( $class, $bibnum, %options ) = @_;

    my $marc = $class->get_marc_biblio( $bibnum, %options );
    return $class->new( $marc, $bibnum );
}

=head2 get_all_biblios_iterator

=cut

sub get_all_biblios_iterator {
    my ( $class, %options ) = @_;

    my $search_terms = {};
    my ( $slice_modulo, $slice_count );
    if ( $options{slice} ) {
        $slice_count  = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
    }

    my $search_options = { columns => [qw/ biblionumber /] };
    if ( $options{desc} ) {
        $search_options->{order_by} = { -desc => 'biblionumber' };
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs       = Koha::Biblios->search(
        $search_terms,
        $search_options
    );

    if ( my $sql = $options{where} ) {
        $rs = $rs->search( \[$sql] );
    }

    my $next_func = sub {

        # Warn and skip bad records, otherwise we break the loop
        while (1) {
            my $row = $rs->next();
            return if !$row;
            my $next = eval {
                my $marc = $row->metadata->record( { embed_items => 1 } );
                $class->new( $marc, $row->biblionumber );
            };
            if ($@) {
                warn sprintf "Something went wrong reading record for biblio %s: %s\n", $row->biblionumber, $@;

=cut

sub get_marc_biblio {
    my ( $class, $bibnum, %options ) = @_;

    my $record = Koha::Biblios->find($bibnum)->metadata->record( { $options{item_data} ? ( embed_items => 1 ) : () } );
      ->metadata->record( { $options{item_data} ? ( embed_items => 1 ) : () } );
    return $record;
}





    return $self;
}


=head2 get_from_authid

    my $auth = Koha::MetadataRecord::Authority->get_from_authid($authid);

=cut

sub get_from_authid {
    my $class       = shift;
    my $authid      = shift;
    my $marcflavour = lc C4::Context->preference("marcflavour");

    my $dbh = C4::Context->dbh;
    my $sth = $dbh->prepare("select authtypecode, marcxml from auth_header where authid=?");
    $sth->execute($authid);
    my ( $authtypecode, $marcxml ) = $sth->fetchrow;
    my $record = eval {
        MARC::Record->new_from_xml(
            StripNonXmlChars($marcxml), 'UTF-8',
            (
                C4::Context->preference("marcflavour") eq "UNIMARC"
                ? "UNIMARCAUTH"
                : C4::Context->preference("marcflavour")
            )
        );
    };
    return if ($@);
    $record->encoding('UTF-8');

    my $self = $class->SUPER::new(
        {
            authid       => $authid,
            authtypecode => $authtypecode,
            schema       => $marcflavour,
            record       => $record
        }
    );

    bless $self, $class;
    return $self;

=cut

sub get_from_breeding {
    my $class            = shift;
    my $import_record_id = shift;
    my $marcflavour      = lc C4::Context->preference("marcflavour");

    my $dbh = C4::Context->dbh;
    my $sth = $dbh->prepare("select marcxml from import_records where import_record_id=? and record_type='auth';");
    $sth->execute($import_record_id);
    my $marcxml = $sth->fetchrow;
    my $record  = eval {
        MARC::Record->new_from_xml(
            StripNonXmlChars($marcxml), 'UTF-8',
            (
                C4::Context->preference("marcflavour") eq "UNIMARC"
                ? "UNIMARCAUTH"
                : C4::Context->preference("marcflavour")
            )
        );
    };
    return if ($@);
    $record->encoding('UTF-8');


    require C4::AuthoritiesMarc;
    my $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);

    my $self = $class->SUPER::new(
        {
            schema       => $marcflavour,
            authtypecode => $authtypecode,
            record       => $record
        }
    );

    bless $self, $class;
    return $self;


sub authorized_heading {
    my ($self) = @_;
    if ( $self->schema =~ m/marc/ ) {
        return Koha::Util::MARC::getAuthorityAuthorizedHeading( $self->record, $self->schema );
    }
    return;
}

=cut

sub get_all_authorities_iterator {
    my ( $self, %options ) = @_;

    my $search_terms = { marcxml => { '!=', undef } };
    my ( $slice_modulo, $slice_count );
    if ( $options{slice} ) {
        $slice_count  = $options{slice}->{count};
    if ($options{slice}) {
        $slice_count = $options{slice}->{count};
        $slice_modulo = $options{slice}->{index};
        $search_terms = {
            '-and' => [

    }

    my $search_options->{columns} = [qw/ authid /];
    if ( $options{desc} ) {
        $search_options->{order_by} = { -desc => 'authid' };
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs       = $schema->resultset('AuthHeader')->search(
      $schema->resultset('AuthHeader')->search(
        $search_terms,
        $search_options
    );

    if ( my $sql = $options{where} ) {
        $rs = $rs->search( \[$sql] );
    }
    my $next_func = sub {

        # Warn and skip bad records, otherwise we break the loop
        while (1) {
            my $row = $rs->next();
            return if !$row;

            my $auth = __PACKAGE__->get_from_authid( $row->authid );
            if ( !$auth ) {
                warn "Something went wrong reading record for authority $row->authid: $@\n";
                next;
            }

Lines 34-39 B<rebuild_elasticsearch.pl> Link Here

(-)a/misc/search_tools/rebuild_elasticsearch.pl (-44 / +62 lines)
34	[B<--desc>]	34	[B<--desc>]
35	[B<-bn\|--bnumber>]	35	[B<-bn\|--bnumber>]
36	[B<-ai\|--authid>]	36	[B<-ai\|--authid>]
		37	[B<-w\|--where SQL>]
37	[B<-p\|--processes>]	38	[B<-p\|--processes>]
38	[B<-v\|--verbose>]	39	[B<-v\|--verbose>]
39	[B<-h\|--help>]	40	[B<-h\|--help>]
Lines 87-92 repeated. Link Here
87	Only index the supplied authority id, mostly for testing purposes. May be	88	Only index the supplied authority id, mostly for testing purposes. May be
88	repeated.	89	repeated.
89		90
		91	=item B<-w\|--where>
		92
		93	Pass some additional SQL to limit the records to be indexed.
		94
90	=item B<-p\|--processes>	95	=item B<-p\|--processes>
91		96
92	Number of processes to use for indexing. This can be used to do more indexing	97	Number of processes to use for indexing. This can be used to do more indexing
Lines 125-137 use Pod::Usage qw( pod2usage ); Link Here
125	use Try::Tiny qw( catch try );	130	use Try::Tiny qw( catch try );
126		131
127	my $verbose = 0;	132	my $verbose = 0;
128	my $commit = 5000;	133	my $commit = 5000;
129	my ($delete, $reset, $help, $man, $processes);	134	my ( $delete, $reset, $help, $man, $processes );
130	my ($index_biblios, $index_authorities);	135	my ( $index_biblios, $index_authorities );
131	my (@biblionumbers,@authids);	136	my ( @biblionumbers, @authids, $where );
132	my $desc;	137	my $desc;
133		138
134	$\|=1; # flushes output	139	$\| = 1; # flushes output
135		140
136	GetOptions(	141	GetOptions(
137	'c\|commit=i' => \$commit,	142	'c\|commit=i' => \$commit,
Lines 142-147 GetOptions( Link Here
142	'desc' => \$desc,	147	'desc' => \$desc,
143	'bn\|bnumber=i' => \@biblionumbers,	148	'bn\|bnumber=i' => \@biblionumbers,
144	'ai\|authid=i' => \@authids,	149	'ai\|authid=i' => \@authids,
		150	'w\|where=s' => \$where,
145	'p\|processes=i' => \$processes,	151	'p\|processes=i' => \$processes,
146	'v\|verbose+' => \$verbose,	152	'v\|verbose+' => \$verbose,
147	'h\|help' => \$help,	153	'h\|help' => \$help,
Lines 149-210 GetOptions( Link Here
149	);	155	);
150		156
151	# Default is to do both	157	# Default is to do both
152	unless ($index_authorities \|\| $index_biblios) {	158	unless ( $index_authorities \|\| $index_biblios ) {
153	$index_authorities = $index_biblios = 1;	159	$index_authorities = $index_biblios = 1;
154	}	160	}
155		161
156	if ($processes && ( @biblionumbers \|\| @authids) ) {	162	if ( $processes && ( @biblionumbers \|\| @authids ) ) {
157	die "Argument p\|processes cannot be combined with bn\|bnumber or ai\|authid";	163	die "Argument p\|processes cannot be combined with bn\|bnumber or ai\|authid";
158	}	164	}
159		165
160	pod2usage(1) if $help;	166	pod2usage(1) if $help;
161	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;	167	pod2usage( -exitstatus => 0, -verbose => 2 ) if $man;
162		168
163	_sanity_check();	169	_sanity_check();
164		170
165	if ($reset){	171	if ($reset) {
166	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;	172	Koha::SearchEngine::Elasticsearch->reset_elasticsearch_mappings;
167	$delete = 1;	173	$delete = 1;
168	}	174	}
169		175
170	_verify_index_state($Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete) if ($index_biblios);	176	_verify_index_state( $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX, $delete ) if ($index_biblios);
171	_verify_index_state($Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete) if ($index_authorities);	177	_verify_index_state( $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX, $delete ) if ($index_authorities);
172		178
173	my $slice_index = 0;	179	my $slice_index = 0;
174	my $slice_count = ( $processes //= 1 );	180	my $slice_count = ( $processes //= 1 );
175	my %iterator_options;	181	my %iterator_options;
176		182
177	if ($slice_count > 1) {	183	if ( $slice_count > 1 ) {
		184
178	# Fire up child processes for processing slices from 2 on. This main process will handle slice 1.	185	# Fire up child processes for processing slices from 2 on. This main process will handle slice 1.
179	$slice_index = 0;	186	$slice_index = 0;
180	for (my $proc = 1; $proc < $slice_count; $proc++) {	187	for ( my $proc = 1 ; $proc < $slice_count ; $proc++ ) {
181	my $pid = fork();	188	my $pid = fork();
182	die "Failed to fork a child process\n" unless defined $pid;	189	die "Failed to fork a child process\n" unless defined $pid;
183	if ($pid == 0) {	190	if ( $pid == 0 ) {
		191
184	# Child process, give it a slice to process	192	# Child process, give it a slice to process
185	$slice_index = $proc;	193	$slice_index = $proc;
186	last;	194	last;
187	}	195	}
188	}	196	}
		197
189	# Fudge the commit count a bit to spread out the Elasticsearch commits	198	# Fudge the commit count a bit to spread out the Elasticsearch commits
190	$commit = 1 + 0.10 $slice_index;	199	$commit = 1 + 0.10 $slice_index;
191	$commit = int( $commit );	200	$commit = int($commit);
192	_log(1, "Processing slice @{[$slice_index + 1]} of $slice_count\n");	201	_log( 1, "Processing slice @{[$slice_index + 1]} of $slice_count\n" );
193	$iterator_options{slice} = { index => $slice_index, count => $slice_count };	202	$iterator_options{slice} = { index => $slice_index, count => $slice_count };
194	}	203	}
195		204
196	if( $desc ){	205	if ($desc) {
197	$iterator_options{desc} = 1;	206	$iterator_options{desc} = 1;
198	}	207	}
199		208
		209	if ($where) {
		210	$iterator_options{where} = $where;
		211	}
		212
200	my $next;	213	my $next;
201	if ($index_biblios) {	214	if ($index_biblios) {
202	_log(1, "Indexing biblios\n");	215	_log( 1, "Indexing biblios\n" );
203	if (@biblionumbers) {	216	if (@biblionumbers) {
204	$next = sub {	217	$next = sub {
205	my $r = shift @biblionumbers;	218	my $r = shift @biblionumbers;
206	return () unless defined $r;	219	return () unless defined $r;
207	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));	220	return ( $r, Koha::BiblioUtils->get_from_biblionumber( $r, item_data => 1 ) );
208	};	221	};
209	} else {	222	} else {
210	my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);	223	my $records = Koha::BiblioUtils->get_all_biblios_iterator(%iterator_options);
Lines 212-227 if ($index_biblios) { Link Here
212	$records->next();	225	$records->next();
213	}	226	}
214	}	227	}
215	_do_reindex($next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX);	228	_do_reindex( $next, $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX );
216	}	229	}
217	if ($index_authorities) {	230	if ($index_authorities) {
218	_log(1, "Indexing authorities\n");	231	_log( 1, "Indexing authorities\n" );
219	if (@authids) {	232	if (@authids) {
220	$next = sub {	233	$next = sub {
221	my $r = shift @authids;	234	my $r = shift @authids;
222	return () unless defined $r;	235	return () unless defined $r;
223	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);	236	my $a = Koha::MetadataRecord::Authority->get_from_authid($r);
224	return ($r, $a);	237	return ( $r, $a );
225	};	238	};
226	} else {	239	} else {
227	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);	240	my $records = Koha::MetadataRecord::Authority->get_all_authorities_iterator(%iterator_options);
Lines 229-240 if ($index_authorities) { Link Here
229	$records->next();	242	$records->next();
230	}	243	}
231	}	244	}
232	_do_reindex($next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX);	245	_do_reindex( $next, $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX );
233	}	246	}
234		247
235	if ($slice_index == 0) {	248	if ( $slice_index == 0 ) {
		249
236	# Main process, wait for children	250	# Main process, wait for children
237	for (my $proc = 1; $proc < $processes; $proc++) {	251	for ( my $proc = 1 ; $proc < $processes ; $proc++ ) {
238	wait();	252	wait();
239	}	253	}
240	}	254	}
Lines 252-272 Checks the index state and recreates it if requested. Link Here
252	sub _verify_index_state {	266	sub _verify_index_state {
253	my ( $index_name, $recreate ) = @_;	267	my ( $index_name, $recreate ) = @_;
254		268
255	_log(1, "Checking state of $index_name index\n");	269	_log( 1, "Checking state of $index_name index\n" );
256	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );	270	my $indexer = Koha::SearchEngine::Elasticsearch::Indexer->new( { index => $index_name } );
257		271
258	if ($recreate) {	272	if ($recreate) {
259	_log(1, "Dropping and recreating $index_name index\n");	273	_log( 1, "Dropping and recreating $index_name index\n" );
260	$indexer->drop_index() if $indexer->index_exists();	274	$indexer->drop_index() if $indexer->index_exists();
261	$indexer->create_index();	275	$indexer->create_index();
262	}	276	} elsif ( !$indexer->index_exists ) {
263	elsif (!$indexer->index_exists) {	277
264	# Create index if does not exist	278	# Create index if does not exist
265	$indexer->create_index();	279	$indexer->create_index();
266	} elsif ($indexer->is_index_status_ok) {	280	} elsif ( $indexer->is_index_status_ok ) {
		281
267	# Update mapping unless index is some kind of problematic state	282	# Update mapping unless index is some kind of problematic state
268	$indexer->update_mappings();	283	$indexer->update_mappings();
269	} elsif ($indexer->is_index_status_recreate_required) {	284	} elsif ( $indexer->is_index_status_recreate_required ) {
270	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;	285	warn qq/Index "$index_name" has status "recreate required", suggesting it should be recreated/;
271	}	286	}
272	}	287	}
Lines 293-299 sub _do_reindex { Link Here
293	my $record = $record->record;	308	my $record = $record->record;
294	$count++;	309	$count++;
295	if ( $verbose == 1 ) {	310	if ( $verbose == 1 ) {
296	_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0);	311	_log( 1, "$count records processed\n" ) if ( $count % 1000 == 0 );
297	} else {	312	} else {
298	_log( 2, "$id\n" );	313	_log( 2, "$id\n" );
299	}	314	}
Lines 303-315 sub _do_reindex { Link Here
303	if ( !( --$commit_count ) ) {	318	if ( !( --$commit_count ) ) {
304	_log( 1, "Committing $commit records...\n" );	319	_log( 1, "Committing $commit records...\n" );
305	my $response;	320	my $response;
306	try{	321	try {
307	$response = $indexer->update_index( \@id_buffer, \@commit_buffer );	322	$response = $indexer->update_index( \@id_buffer, \@commit_buffer );
308	_handle_response($response);	323	_handle_response($response);
309	_log( 1, "Commit complete\n" );	324	_log( 1, "Commit complete\n" );
310	} catch {	325	} catch {
311	_log(1,"Elasticsearch exception thrown: ".$_->type."\n");	326	_log( 1, "Elasticsearch exception thrown: " . $_->type . "\n" );
312	_log(2,"Details: ".$_->details."\n");	327	_log( 2, "Details: " . $_->details . "\n" );
313	};	328	};
314	$commit_count = $commit;	329	$commit_count = $commit;
315	@id_buffer = ();	330	@id_buffer = ();
Lines 333-338 Checks some basic stuff to ensure that it's sane before we start. Link Here
333	=cut	348	=cut
334		349
335	sub _sanity_check {	350	sub _sanity_check {
		351
336	# Do we have an elasticsearch block defined?	352	# Do we have an elasticsearch block defined?
337	my $conf = C4::Context->config('elasticsearch');	353	my $conf = C4::Context->config('elasticsearch');
338	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );	354	die "No 'elasticsearch' block is defined in koha-conf.xml.\n" if ( !$conf );
Lines 346-359 Parse the return from update_index and display errors depending on verbosity of Link Here
346		362
347	sub _handle_response {	363	sub _handle_response {
348	my ($response) = @_;	364	my ($response) = @_;
349	if( $response->{errors} eq 'true' ){	365	if ( $response->{errors} eq 'true' ) {
350	_log( 1, "There were errors during indexing\n" );	366	_log( 1, "There were errors during indexing\n" );
351	if ( $verbose > 1 ){	367	if ( $verbose > 1 ) {
352	foreach my $item (@{$response->{items}}){	368	foreach my $item ( @{ $response->{items} } ) {
353	next unless defined $item->{index}->{error};	369	next unless defined $item->{index}->{error};
354	print "Record #" . $item->{index}->{_id} . " " .	370	print "Record #"
355	$item->{index}->{error}->{reason} . " (" . $item->{index}->{error}->{type} . ") : " .	371	. $item->{index}->{_id} . " "
356	$item->{index}->{error}->{caused_by}->{type} . " (" . $item->{index}->{error}->{caused_by}->{reason} . ")\n";	372	. $item->{index}->{error}->{reason} . " ("
		373	. $item->{index}->{error}->{type} . ") : "
		374	. $item->{index}->{error}->{caused_by}->{type} . " ("
		375	. $item->{index}->{error}->{caused_by}->{reason} . ")\n";
357	}	376	}
358	}	377	}
359	}	378	}
Lines 371-377 include a trailing newline automatically. Link Here
371	=cut	390	=cut
372		391
373	sub _log {	392	sub _log {
374	my ($level, $msg) = @_;	393	my ( $level, $msg ) = @_;
375		394
376	print "[$$] $msg" if ($verbose >= $level);	395	print "[$$] $msg" if ( $verbose >= $level );
377	}	396	}
378	-

Return to bug 35345

Lines 34-40 use Koha::MetadataIterator; Link Here

(-)a/Koha/BiblioUtils.pm (-19 / +24 lines)
34	use Koha::Database;	34	use Koha::Database;
35	use Modern::Perl;	35	use Modern::Perl;
36		36
37
38	use base qw(Koha::MetadataRecord);	37	use base qw(Koha::MetadataRecord);
39		38
40	__PACKAGE__->mk_accessors(qw( record schema id datatype ));	39	__PACKAGE__->mk_accessors(qw( record schema id datatype ));
Lines 51-58 the biblionumber can be provided too. Link Here
51	=cut	50	=cut
52		51
53	sub new {	52	sub new {
54	my $class = shift;	53	my $class = shift;
55	my $record = shift;	54	my $record = shift;
56	my $biblionumber = shift;	55	my $biblionumber = shift;
57		56
58	my $self = $class->SUPER::new(	57	my $self = $class->SUPER::new(
Lines 89-98 It will return C<undef> if the biblio doesn't exist. Link Here
89	=cut	88	=cut
90		89
91	sub get_from_biblionumber {	90	sub get_from_biblionumber {
92	my ($class, $bibnum, %options) = @_;	91	my ( $class, $bibnum, %options ) = @_;
93		92
94	my $marc = $class->get_marc_biblio($bibnum, %options);	93	my $marc = $class->get_marc_biblio( $bibnum, %options );
95	return $class->new($marc, $bibnum);	94	return $class->new( $marc, $bibnum );
96	}	95	}
97		96
98	=head2 get_all_biblios_iterator	97	=head2 get_all_biblios_iterator
Lines 120-153 records instead of all. Link Here
120	=cut	119	=cut
121		120
122	sub get_all_biblios_iterator {	121	sub get_all_biblios_iterator {
123	my ($class, %options) = @_;	122	my ( $class, %options ) = @_;
124		123
125	my $search_terms = {};	124	my $search_terms = {};
126	my ($slice_modulo, $slice_count);	125	my ( $slice_modulo, $slice_count );
127	if ($options{slice}) {	126	if ( $options{slice} ) {
128	$slice_count = $options{slice}->{count};	127	$slice_count = $options{slice}->{count};
129	$slice_modulo = $options{slice}->{index};	128	$slice_modulo = $options{slice}->{index};
130	$search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];	129	$search_terms = \[ 'mod(biblionumber, ?) = ?', $slice_count, $slice_modulo ];
131	}	130	}
132		131
133	my $search_options = { columns => [qw/ biblionumber /] };	132	my $search_options = { columns => [qw/ biblionumber /] };
134	if ( $options{desc} ){	133	if ( $options{desc} ) {
135	$search_options->{order_by} = { -desc => 'biblionumber' };	134	$search_options->{order_by} = { -desc => 'biblionumber' };
136	}	135	}
137		136
138	my $database = Koha::Database->new();	137	my $database = Koha::Database->new();
139	my $schema = $database->schema();	138	my $schema = $database->schema();
140	my $rs = Koha::Biblios->search(	139	my $rs = Koha::Biblios->search(
141	$search_terms,	140	$search_terms,
142	$search_options );	141	$search_options
		142	);
		143
		144	if ( my $sql = $options{where} ) {
		145	$rs = $rs->search( \[$sql] );
		146	}
		147
143	my $next_func = sub {	148	my $next_func = sub {
		149
144	# Warn and skip bad records, otherwise we break the loop	150	# Warn and skip bad records, otherwise we break the loop
145	while (1) {	151	while (1) {
146	my $row = $rs->next();	152	my $row = $rs->next();
147	return if !$row;	153	return if !$row;
148	my $next = eval {	154	my $next = eval {
149	my $marc = $row->metadata->record({ embed_items => 1 });	155	my $marc = $row->metadata->record( { embed_items => 1 } );
150	$class->new($marc, $row->biblionumber);	156	$class->new( $marc, $row->biblionumber );
151	};	157	};
152	if ($@) {	158	if ($@) {
153	warn sprintf "Something went wrong reading record for biblio %s: %s\n", $row->biblionumber, $@;	159	warn sprintf "Something went wrong reading record for biblio %s: %s\n", $row->biblionumber, $@;
Lines 180-189 If set to true, item data is embedded in the record. Default is to not do this. Link Here
180	=cut	186	=cut
181		187
182	sub get_marc_biblio {	188	sub get_marc_biblio {
183	my ($class, $bibnum, %options) = @_;	189	my ( $class, $bibnum, %options ) = @_;
184		190
185	my $record = Koha::Biblios->find($bibnum)	191	my $record = Koha::Biblios->find($bibnum)->metadata->record( { $options{item_data} ? ( embed_items => 1 ) : () } );
186	->metadata->record( { $options{item_data} ? ( embed_items => 1 ) : () } );
187	return $record;	192	return $record;
188	}	193	}
189		194

Lines 67-73 sub new { Link Here

(-)a/Koha/MetadataRecord/Authority.pm (-37 / +62 lines)
67	return $self;	67	return $self;
68	}	68	}
69		69
70
71	=head2 get_from_authid	70	=head2 get_from_authid
72		71
73	my $auth = Koha::MetadataRecord::Authority->get_from_authid($authid);	72	my $auth = Koha::MetadataRecord::Authority->get_from_authid($authid);
Lines 80-102 unfortunate but unavoidable fact. Link Here
80	=cut	79	=cut
81		80
82	sub get_from_authid {	81	sub get_from_authid {
83	my $class = shift;	82	my $class = shift;
84	my $authid = shift;	83	my $authid = shift;
85	my $marcflavour = lc C4::Context->preference("marcflavour");	84	my $marcflavour = lc C4::Context->preference("marcflavour");
86		85
87	my $dbh=C4::Context->dbh;	86	my $dbh = C4::Context->dbh;
88	my $sth=$dbh->prepare("select authtypecode, marcxml from auth_header where authid=?");	87	my $sth = $dbh->prepare("select authtypecode, marcxml from auth_header where authid=?");
89	$sth->execute($authid);	88	$sth->execute($authid);
90	my ($authtypecode, $marcxml) = $sth->fetchrow;	89	my ( $authtypecode, $marcxml ) = $sth->fetchrow;
91	my $record=eval {MARC::Record->new_from_xml(StripNonXmlChars($marcxml),'UTF-8',	90	my $record = eval {
92	(C4::Context->preference("marcflavour") eq "UNIMARC"?"UNIMARCAUTH":C4::Context->preference("marcflavour")))};	91	MARC::Record->new_from_xml(
		92	StripNonXmlChars($marcxml), 'UTF-8',
		93	(
		94	C4::Context->preference("marcflavour") eq "UNIMARC"
		95	? "UNIMARCAUTH"
		96	: C4::Context->preference("marcflavour")
		97	)
		98	);
		99	};
93	return if ($@);	100	return if ($@);
94	$record->encoding('UTF-8');	101	$record->encoding('UTF-8');
95		102
96	my $self = $class->SUPER::new( { authid => $authid,	103	my $self = $class->SUPER::new(
97	authtypecode => $authtypecode,	104	{
98	schema => $marcflavour,	105	authid => $authid,
99	record => $record });	106	authtypecode => $authtypecode,
		107	schema => $marcflavour,
		108	record => $record
		109	}
		110	);
100		111
101	bless $self, $class;	112	bless $self, $class;
102	return $self;	113	return $self;
Lines 111-126 Create the Koha::MetadataRecord::Authority object associated with the provided a Link Here
111	=cut	122	=cut
112		123
113	sub get_from_breeding {	124	sub get_from_breeding {
114	my $class = shift;	125	my $class = shift;
115	my $import_record_id = shift;	126	my $import_record_id = shift;
116	my $marcflavour = lc C4::Context->preference("marcflavour");	127	my $marcflavour = lc C4::Context->preference("marcflavour");
117		128
118	my $dbh=C4::Context->dbh;	129	my $dbh = C4::Context->dbh;
119	my $sth=$dbh->prepare("select marcxml from import_records where import_record_id=? and record_type='auth';");	130	my $sth = $dbh->prepare("select marcxml from import_records where import_record_id=? and record_type='auth';");
120	$sth->execute($import_record_id);	131	$sth->execute($import_record_id);
121	my $marcxml = $sth->fetchrow;	132	my $marcxml = $sth->fetchrow;
122	my $record=eval {MARC::Record->new_from_xml(StripNonXmlChars($marcxml),'UTF-8',	133	my $record = eval {
123	(C4::Context->preference("marcflavour") eq "UNIMARC"?"UNIMARCAUTH":C4::Context->preference("marcflavour")))};	134	MARC::Record->new_from_xml(
		135	StripNonXmlChars($marcxml), 'UTF-8',
		136	(
		137	C4::Context->preference("marcflavour") eq "UNIMARC"
		138	? "UNIMARCAUTH"
		139	: C4::Context->preference("marcflavour")
		140	)
		141	);
		142	};
124	return if ($@);	143	return if ($@);
125	$record->encoding('UTF-8');	144	$record->encoding('UTF-8');
126		145
Lines 131-140 sub get_from_breeding { Link Here
131	require C4::AuthoritiesMarc;	150	require C4::AuthoritiesMarc;
132	my $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);	151	my $authtypecode = C4::AuthoritiesMarc::GuessAuthTypeCode($record);
133		152
134	my $self = $class->SUPER::new( {	153	my $self = $class->SUPER::new(
135	schema => $marcflavour,	154	{
136	authtypecode => $authtypecode,	155	schema => $marcflavour,
137	record => $record });	156	authtypecode => $authtypecode,
		157	record => $record
		158	}
		159	);
138		160
139	bless $self, $class;	161	bless $self, $class;
140	return $self;	162	return $self;
Lines 142-149 sub get_from_breeding { Link Here
142		164
143	sub authorized_heading {	165	sub authorized_heading {
144	my ($self) = @_;	166	my ($self) = @_;
145	if ($self->schema =~ m/marc/) {	167	if ( $self->schema =~ m/marc/ ) {
146	return Koha::Util::MARC::getAuthorityAuthorizedHeading($self->record, $self->schema);	168	return Koha::Util::MARC::getAuthorityAuthorizedHeading( $self->record, $self->schema );
147	}	169	}
148	return;	170	return;
149	}	171	}
Lines 173-186 records instead of all. Link Here
173	=cut	195	=cut
174		196
175	sub get_all_authorities_iterator {	197	sub get_all_authorities_iterator {
176	my ($self, %options) = @_;	198	my ( $self, %options ) = @_;
177		199
178	my $search_terms = {	200	my $search_terms = { marcxml => { '!=', undef } };
179	marcxml => { '!=', undef }	201	my ( $slice_modulo, $slice_count );
180	};	202	if ( $options{slice} ) {
181	my ($slice_modulo, $slice_count);	203	$slice_count = $options{slice}->{count};
182	if ($options{slice}) {
183	$slice_count = $options{slice}->{count};
184	$slice_modulo = $options{slice}->{index};	204	$slice_modulo = $options{slice}->{index};
185	$search_terms = {	205	$search_terms = {
186	'-and' => [	206	'-and' => [
Lines 191-214 sub get_all_authorities_iterator { Link Here
191	}	211	}
192		212
193	my $search_options->{columns} = [qw/ authid /];	213	my $search_options->{columns} = [qw/ authid /];
194	if ($options{desc}) {	214	if ( $options{desc} ) {
195	$search_options->{order_by} = { -desc => 'authid' };	215	$search_options->{order_by} = { -desc => 'authid' };
196	}	216	}
197		217
198	my $database = Koha::Database->new();	218	my $database = Koha::Database->new();
199	my $schema = $database->schema();	219	my $schema = $database->schema();
200	my $rs =	220	my $rs = $schema->resultset('AuthHeader')->search(
201	$schema->resultset('AuthHeader')->search(
202	$search_terms,	221	$search_terms,
203	$search_options);	222	$search_options
		223	);
		224
		225	if ( my $sql = $options{where} ) {
		226	$rs = $rs->search( \[$sql] );
		227	}
204	my $next_func = sub {	228	my $next_func = sub {
		229
205	# Warn and skip bad records, otherwise we break the loop	230	# Warn and skip bad records, otherwise we break the loop
206	while (1) {	231	while (1) {
207	my $row = $rs->next();	232	my $row = $rs->next();
208	return if !$row;	233	return if !$row;
209		234
210	my $auth = __PACKAGE__->get_from_authid($row->authid);	235	my $auth = __PACKAGE__->get_from_authid( $row->authid );
211	if (!$auth) {	236	if ( !$auth ) {
212	warn "Something went wrong reading record for authority $row->authid: $@\n";	237	warn "Something went wrong reading record for authority $row->authid: $@\n";
213	next;	238	next;
214	}	239	}