Attachment #70606 for bug #19884

View | Details | Raw Unified | Return to bug 19884
Collapse All | Expand All




    return wantarray ? @{$mss->{$kohafield}} : $mss->{$kohafield}->[0];
}


sub GetMarcBiblio {
    my ($params) = @_;

    if (not defined $params) {
        carp 'GetMarcBiblio called without parameters';
        return;
    }
    my $biblionumber = $params->{biblionumber};

    if (not defined $biblionumber) {
        carp 'GetMarcBiblio called with undefined biblionumber';
        return;
    }
    delete $params->{biblionumber};
    $params->{biblionumbers} = [$biblionumber];
    my ($biblio) = GetMarcBiblios($params);
    return $biblio;
}

=head2 GetMarcBiblio

  my $record = GetMarcBiblio({


=cut

sub GetMarcBiblios {
    my ($params) = @_;

    if (not defined $params) {
        carp 'GetMarcBiblios called without parameters';
        return;
    }

    my $biblionumbers = $params->{biblionumbers};
    my $embeditems   = $params->{embed_items} || 0;
    my $opac         = $params->{opac} || 0;

    if (not defined $biblionumbers) {
        carp 'GetMarcBiblio called with undefined biblionumbers';
        return;
    }

    my $in_placeholders = join ', ', (('?') x @{$biblionumbers});
    my $dbh = C4::Context->dbh;
    my $sth = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE biblionumber IN ($in_placeholders)");
    $sth->execute(@{$biblionumbers});

    my %marc_records;
    my ($biblionumber, $biblioitemnumber);
    my $marc_flavour = C4::Context->preference('marcflavour');
    while (my $biblio_keys = $sth->fetchrow_arrayref) {
        ($biblionumber, $biblioitemnumber) = @{$biblio_keys};

        my $marcxml = GetXmlBiblio( $biblionumber );
        $marcxml = StripNonXmlChars( $marcxml );
        my $frameworkcode = GetFrameworkCode($biblionumber);
        MARC::File::XML->default_record_format( $marc_flavour );
        my $record = MARC::Record->new();

        if ($marcxml) {
            $record = eval {
                MARC::Record::new_from_xml( $marcxml, "utf8", $marc_flavour);
            };
            if ($@) { warn " problem with :$biblionumber : $@ \n$marcxml"; }
            next unless $record;

            C4::Biblio::_koha_marc_update_bib_ids(
                $record,
                $frameworkcode,
                $biblionumber,
                $biblioitemnumber
            );
            $marc_records{$biblionumber} = $record;
        }
    }

    if ($embeditems) {
        C4::Biblio::EmbedItemsInMarcBiblios(
            \%marc_records,
            undef,
            $opac
        );
    }
    return wantarray ? values %marc_records : \%marc_records;
}

=head2 GetXmlBiblio

        carp 'EmbedItemsInMarcBiblio: No MARC record passed';
        return;
    }
    my ($marc_with_items) = EmbedItemsInMarcBiblios(
        { $biblionumber => $marc },
        $itemnumbers,
        $opac
    );
    return $marc_with_items;
}

sub EmbedItemsInMarcBiblios {
    my ($marc_records, $itemnumbers, $opac) = @_;

    $itemnumbers = [] unless defined $itemnumbers;
    # exists for improved performance? Probably..
    my $frameworkcode = GetFrameworkCode($biblionumber);
    _strip_item_fields($marc, $frameworkcode);

    # ... and embed the current items
    my $dbh = C4::Context->dbh;

    my @biblionumbers = keys %{$marc_records};
    my $in_placeholders = join ', ', (('?') x @biblionumbers);

    my $biblio_itemnumbers = $dbh->selectcol_arrayref(
        "SELECT itemnumber FROM items WHERE biblionumber IN ($in_placeholders)",
        undef,
        @biblionumbers
    );

    # Remove invalid item numbers by intersecting with all valid item numbers
    # if $itemnumbers argument was provided

    # TODO: MAKE SURE THERE IS A TEST FOR THIS
    if(@{$itemnumbers}) {
        my %h = map { $_ => undef } @{$itemnumbers};
        $itemnumbers = [grep { exists $h{$_} } @{$biblio_itemnumbers}];
    }
    else {
        $itemnumbers = $biblio_itemnumbers;
    }
    # If no item numbers then no point in continuing
    return unless (@{$itemnumbers});

    require C4::Items;

    my $items = C4::Items::GetItems($itemnumbers);
    # We have now fetched all items, time to partion by biblionumber
    my %items_by_biblionumber = (map { ($_, []) } @biblionumbers);
    foreach my $item (@{$items}) {
        push @{$items_by_biblionumber{$item->{biblionumber}}}, $item;
    }

    foreach my $biblionumber (@biblionumbers) {
        my $marc_record = $marc_records->{$biblionumber};
        my $items = $items_by_biblionumber{$biblionumber};
        # Could this be moved lower down and run only when items
        # exists for improved performance? Probably..
        my $frameworkcode = GetFrameworkCode($biblionumber);
        _strip_item_fields($marc_record, $frameworkcode);

        if ($opachiddenitems) {
            my %hidden_items = map { $_ => undef } C4::Items::GetHiddenItemnumbers(@{$items});
            # Reduce items to non hidden items
            $items = [grep { !(exists $hidden_items{$_->{itemnumber}}) } @{$items}];
        }

        my ($itemtag) = GetMarcFromKohaField("items.itemnumber", $frameworkcode);
        my @item_fields;
        foreach my $item (@{$items}) {
            my $item_marc = C4::Items::GetMarcItem($biblionumber, $item);
            push @item_fields, $item_marc->field($itemtag);
        }
        $marc_record->append_fields(@item_fields);
    }
    return $marc_records;
}

=head1 INTERNAL FUNCTIONS




=cut

sub get_all_biblios_iterator {
    my ($self, $params) = @_;
    my $batched;
    my $batch_size;
    if ($params) {
        $batched //= $params->{batched};
        $batch_size = $params->{batch_size} || 2000;
    }

    my $database = Koha::Database->new();
    my $schema   = $database->schema();
    my $rs = $schema->resultset('Biblio')->search(
        {},
        { columns => [qw/ biblionumber /] }
    );

    my $next_func;
    if ($batched) {
        my @marc_records_batch;
        $next_func = sub {
            unless (@marc_records_batch) {
                #Batch empty, nead to buffer up more records
                my $limit = $batch_size;
                my @biblionumbers;
                while ($limit--) {
                    my $row = $rs->next();
                    last if !$row;
                    push @biblionumbers, $row->biblionumber;
                }
                if (@biblionumbers) {
                    my $marc_records = C4::Biblio::GetMarcBiblios({
                        biblionumbers => \@biblionumbers,
                        embed_items => 1
                    });
                    while (my ($biblionumber, $marc_record) = each %{$marc_records}) {
                        my $next = __PACKAGE__->new($marc_record, $biblionumber);
                        push @marc_records_batch, $next;
                    }
                }
            }
            return pop @marc_records_batch;
        };
    }
    else {
        $next_func = sub {
            # Warn and skip bad records, otherwise we break the loop
            while (1) {
                my $row = $rs->next();
                return if !$row;
                my $marc = C4::Biblio::GetMarcBiblio({
                    biblionumber => $row->biblionumber,
                    embed_items => 1
                });
                my $next = eval {
                    __PACKAGE__->new($marc, $row->biblionumber);
                };
                if ($@) {
                    warn "Something went wrong reading record for biblio $row->biblionumber: $@\n";
                    next;
                }
                return $next;
            }
        };
    }
    return Koha::MetadataIterator->new($next_func);
}


Lines 106-113 sub search_unblessed { Link Here

(-)a/Koha/Items.pm (-2 / +2 lines)
106	push @{$items}, $item;	106	push @{$items}, $item;
107	}	107	}
108	elsif (ref($values) eq 'ARRAY') {	108	elsif (ref($values) eq 'ARRAY') {
109	my $in = join ', ', (('?') x @{$values});	109	my $in_placeholders = join ', ', (('?') x @{$values});
110	my $sth = C4::Context->dbh->prepare(qq{SELECT * FROM items WHERE $field IN($in)});	110	my $sth = C4::Context->dbh->prepare(qq{SELECT * FROM items WHERE $field IN($in_placeholders)});
111	$sth->execute(@{$values});	111	$sth->execute(@{$values});
112	while (my $item = $sth->fetchrow_hashref) {	112	while (my $item = $sth->fetchrow_hashref) {
113	push @{$items}, $item;	113	push @{$items}, $item;

Lines 131-137 if ($index_biblios) { Link Here

(-)a/misc/search_tools/rebuild_elastic_search.pl (-2 / +1 lines)
131	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));	131	return ($r, Koha::BiblioUtils->get_from_biblionumber($r, item_data => 1 ));
132	};	132	};
133	} else {	133	} else {
134	my $records = Koha::BiblioUtils->get_all_biblios_iterator();	134	my $records = Koha::BiblioUtils->get_all_biblios_iterator({ batched => 1 });
135	$next = sub {	135	$next = sub {
136	$records->next();	136	$records->next();
137	}	137	}
138	-

Return to bug 19884

Lines 1161-1166 sub GetMarcSubfieldStructureFromKohaField { Link Here

(-)a/C4/Biblio.pm (-56 / +118 lines)
1161	return wantarray ? @{$mss->{$kohafield}} : $mss->{$kohafield}->[0];	1161	return wantarray ? @{$mss->{$kohafield}} : $mss->{$kohafield}->[0];
1162	}	1162	}
1163		1163
		1164
		1165	sub GetMarcBiblio {
		1166	my ($params) = @_;
		1167
		1168	if (not defined $params) {
		1169	carp 'GetMarcBiblio called without parameters';
		1170	return;
		1171	}
		1172	my $biblionumber = $params->{biblionumber};
		1173
		1174	if (not defined $biblionumber) {
		1175	carp 'GetMarcBiblio called with undefined biblionumber';
		1176	return;
		1177	}
		1178	delete $params->{biblionumber};
		1179	$params->{biblionumbers} = [$biblionumber];
		1180	my ($biblio) = GetMarcBiblios($params);
		1181	return $biblio;
		1182	}
		1183
1164	=head2 GetMarcBiblio	1184	=head2 GetMarcBiblio
1165		1185
1166	my $record = GetMarcBiblio({	1186	my $record = GetMarcBiblio({
Lines 1194-1245 OpacHiddenItems to be applied. Link Here
1194		1214
1195	=cut	1215	=cut
1196		1216
1197	sub GetMarcBiblio {	1217	sub GetMarcBiblios {
1198	my ($params) = @_;	1218	my ($params) = @_;
1199		1219
1200	if (not defined $params) {	1220	if (not defined $params) {
1201	carp 'GetMarcBiblio called without parameters';	1221	carp 'GetMarcBiblios called without parameters';
1202	return;	1222	return;
1203	}	1223	}
1204		1224
1205	my $biblionumber = $params->{biblionumber};	1225	my $biblionumbers = $params->{biblionumbers};
1206	my $embeditems = $params->{embed_items} \|\| 0;	1226	my $embeditems = $params->{embed_items} \|\| 0;
1207	my $opac = $params->{opac} \|\| 0;	1227	my $opac = $params->{opac} \|\| 0;
1208		1228
1209	if (not defined $biblionumber) {	1229	if (not defined $biblionumbers) {
1210	carp 'GetMarcBiblio called with undefined biblionumber';	1230	carp 'GetMarcBiblio called with undefined biblionumbers';
1211	return;	1231	return;
1212	}	1232	}
1213		1233
1214	my $dbh = C4::Context->dbh;	1234	my $in_placeholders = join ', ', (('?') x @{$biblionumbers});
1215	my $sth = $dbh->prepare("SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=? ");	1235	my $dbh = C4::Context->dbh;
1216	$sth->execute($biblionumber);	1236	my $sth = $dbh->prepare("SELECT biblionumber, biblioitemnumber FROM biblioitems WHERE biblionumber IN ($in_placeholders)");
1217	my $row = $sth->fetchrow_hashref;	1237	$sth->execute(@{$biblionumbers});
1218	my $biblioitemnumber = $row->{'biblioitemnumber'};	1238
1219	my $marcxml = GetXmlBiblio( $biblionumber );	1239	my %marc_records;
1220	$marcxml = StripNonXmlChars( $marcxml );	1240	my ($biblionumber, $biblioitemnumber);
1221	my $frameworkcode = GetFrameworkCode($biblionumber);	1241	my $marc_flavour = C4::Context->preference('marcflavour');
1222	MARC::File::XML->default_record_format( C4::Context->preference('marcflavour') );	1242	while (my $biblio_keys = $sth->fetchrow_arrayref) {
1223	my $record = MARC::Record->new();	1243	($biblionumber, $biblioitemnumber) = @{$biblio_keys};
1224		1244
1225	if ($marcxml) {	1245	my $marcxml = GetXmlBiblio( $biblionumber );
1226	$record = eval {	1246	$marcxml = StripNonXmlChars( $marcxml );
1227	MARC::Record::new_from_xml( $marcxml, "utf8",	1247	my $frameworkcode = GetFrameworkCode($biblionumber);
1228	C4::Context->preference('marcflavour') );	1248	MARC::File::XML->default_record_format( $marc_flavour );
1229	};	1249	my $record = MARC::Record->new();
1230	if ($@) { warn " problem with :$biblionumber : $@ \n$marcxml"; }	1250
1231	return unless $record;	1251	if ($marcxml) {
1232		1252	$record = eval {
1233	C4::Biblio::_koha_marc_update_bib_ids( $record, $frameworkcode, $biblionumber,	1253	MARC::Record::new_from_xml( $marcxml, "utf8", $marc_flavour);
1234	$biblioitemnumber );	1254	};
1235	C4::Biblio::EmbedItemsInMarcBiblio( $record, $biblionumber, undef, $opac )	1255	if ($@) { warn " problem with :$biblionumber : $@ \n$marcxml"; }
1236	if ($embeditems);	1256	next unless $record;
1237		1257
1238	return $record;	1258	C4::Biblio::_koha_marc_update_bib_ids(
		1259	$record,
		1260	$frameworkcode,
		1261	$biblionumber,
		1262	$biblioitemnumber
		1263	);
		1264	$marc_records{$biblionumber} = $record;
		1265	}
1239	}	1266	}
1240	else {	1267
1241	return;	1268	if ($embeditems) {
		1269	C4::Biblio::EmbedItemsInMarcBiblios(
		1270	\%marc_records,
		1271	undef,
		1272	$opac
		1273	);
1242	}	1274	}
		1275	return wantarray ? values %marc_records : \%marc_records;
1243	}	1276	}
1244		1277
1245	=head2 GetXmlBiblio	1278	=head2 GetXmlBiblio
Lines 2810-2837 sub EmbedItemsInMarcBiblio { Link Here
2810	carp 'EmbedItemsInMarcBiblio: No MARC record passed';	2843	carp 'EmbedItemsInMarcBiblio: No MARC record passed';
2811	return;	2844	return;
2812	}	2845	}
2813	$itemnumbers = [] unless defined $itemnumbers;	2846	my ($marc_with_items) = EmbedItemsInMarcBiblios(
		2847	{ $biblionumber => $marc },
		2848	$itemnumbers,
		2849	$opac
		2850	);
		2851	return $marc_with_items;
		2852	}
		2853
		2854	sub EmbedItemsInMarcBiblios {
		2855	my ($marc_records, $itemnumbers, $opac) = @_;
2814		2856
2815	# Could this be moved lower down and run only when items	2857	$itemnumbers = [] unless defined $itemnumbers;
2816	# exists for improved performance? Probably..
2817	my $frameworkcode = GetFrameworkCode($biblionumber);
2818	_strip_item_fields($marc, $frameworkcode);
2819		2858
2820	# ... and embed the current items	2859	# ... and embed the current items
2821	my $dbh = C4::Context->dbh;	2860	my $dbh = C4::Context->dbh;
2822	my $sth = $dbh->prepare("SELECT itemnumber FROM items WHERE biblionumber = ?");	2861
2823	$sth->execute($biblionumber);	2862	my @biblionumbers = keys %{$marc_records};
2824	my @all_itemnumbers;	2863	my $in_placeholders = join ', ', (('?') x @biblionumbers);
2825	while ( my ($itemnumber) = $sth->fetchrow_array ) {	2864
2826	push @all_itemnumbers, $itemnumber;	2865	my $biblio_itemnumbers = $dbh->selectcol_arrayref(
2827	}	2866	"SELECT itemnumber FROM items WHERE biblionumber IN ($in_placeholders)",
		2867	undef,
		2868	@biblionumbers
		2869	);
		2870
2828	# Remove invalid item numbers by intersecting with all valid item numbers	2871	# Remove invalid item numbers by intersecting with all valid item numbers
2829	if(@$itemnumbers) {	2872	# if $itemnumbers argument was provided
		2873
		2874	# TODO: MAKE SURE THERE IS A TEST FOR THIS
		2875	if(@{$itemnumbers}) {
2830	my %h = map { $_ => undef } @{$itemnumbers};	2876	my %h = map { $_ => undef } @{$itemnumbers};
2831	$itemnumbers = [grep { exists $h{$_} } @all_itemnumbers];	2877	$itemnumbers = [grep { exists $h{$_} } @{$biblio_itemnumbers}];
2832	}	2878	}
2833	else {	2879	else {
2834	$itemnumbers = \@all_itemnumbers;	2880	$itemnumbers = $biblio_itemnumbers;
2835	}	2881	}
2836	# If no item numbers then no point in continuing	2882	# If no item numbers then no point in continuing
2837	return unless (@{$itemnumbers});	2883	return unless (@{$itemnumbers});
Lines 2841-2859 sub EmbedItemsInMarcBiblio { Link Here
2841	require C4::Items;	2887	require C4::Items;
2842		2888
2843	my $items = C4::Items::GetItems($itemnumbers);	2889	my $items = C4::Items::GetItems($itemnumbers);
2844	if ($opachiddenitems) {	2890	# We have now fetched all items, time to partion by biblionumber
2845	my %hidden_items = map { $_ => undef } C4::Items::GetHiddenItemnumbers(@{$items});	2891	my %items_by_biblionumber = (map { ($_, []) } @biblionumbers);
2846	# Reduce items to non hidden items	2892	foreach my $item (@{$items}) {
2847	$items = [grep { !(exists $hidden_items{$_->{itemnumber}}) } @{$items}];	2893	push @{$items_by_biblionumber{$item->{biblionumber}}}, $item;
2848	}	2894	}
2849		2895
2850	my ($itemtag) = GetMarcFromKohaField("items.itemnumber", $frameworkcode);	2896	foreach my $biblionumber (@biblionumbers) {
2851	my @item_fields;	2897	my $marc_record = $marc_records->{$biblionumber};
2852	foreach my $item (@{$items}) {	2898	my $items = $items_by_biblionumber{$biblionumber};
2853	my $item_marc = C4::Items::GetMarcItem($biblionumber, $item);	2899	# Could this be moved lower down and run only when items
2854	push @item_fields, $item_marc->field($itemtag);	2900	# exists for improved performance? Probably..
		2901	my $frameworkcode = GetFrameworkCode($biblionumber);
		2902	_strip_item_fields($marc_record, $frameworkcode);
		2903
		2904	if ($opachiddenitems) {
		2905	my %hidden_items = map { $_ => undef } C4::Items::GetHiddenItemnumbers(@{$items});
		2906	# Reduce items to non hidden items
		2907	$items = [grep { !(exists $hidden_items{$_->{itemnumber}}) } @{$items}];
		2908	}
		2909
		2910	my ($itemtag) = GetMarcFromKohaField("items.itemnumber", $frameworkcode);
		2911	my @item_fields;
		2912	foreach my $item (@{$items}) {
		2913	my $item_marc = C4::Items::GetMarcItem($biblionumber, $item);
		2914	push @item_fields, $item_marc->field($itemtag);
		2915	}
		2916	$marc_record->append_fields(@item_fields);
2855	}	2917	}
2856	$marc->append_fields(@item_fields);	2918	return $marc_records;
2857	}	2919	}
2858		2920
2859	=head1 INTERNAL FUNCTIONS	2921	=head1 INTERNAL FUNCTIONS

Lines 111-139 The iterator is a Koha::MetadataIterator object. Link Here

(-)a/Koha/BiblioUtils.pm (-20 / +60 lines)
111	=cut	111	=cut
112		112
113	sub get_all_biblios_iterator {	113	sub get_all_biblios_iterator {
		114	my ($self, $params) = @_;
		115	my $batched;
		116	my $batch_size;
		117	if ($params) {
		118	$batched //= $params->{batched};
		119	$batch_size = $params->{batch_size} \|\| 2000;
		120	}
		121
114	my $database = Koha::Database->new();	122	my $database = Koha::Database->new();
115	my $schema = $database->schema();	123	my $schema = $database->schema();
116	my $rs =	124	my $rs = $schema->resultset('Biblio')->search(
117	$schema->resultset('Biblio')->search( {},	125	{},
118	{ columns => [qw/ biblionumber /] } );	126	{ columns => [qw/ biblionumber /] }
119	my $next_func = sub {	127	);
120	# Warn and skip bad records, otherwise we break the loop	128
121	while (1) {	129	my $next_func;
122	my $row = $rs->next();	130	if ($batched) {
123	return if !$row;	131	my @marc_records_batch;
124	my $marc = C4::Biblio::GetMarcBiblio({	132	$next_func = sub {
125	biblionumber => $row->biblionumber,	133	unless (@marc_records_batch) {
126	embed_items => 1 });	134	#Batch empty, nead to buffer up more records
127	my $next = eval {	135	my $limit = $batch_size;
128	__PACKAGE__->new($marc, $row->biblionumber);	136	my @biblionumbers;
129	};	137	while ($limit--) {
130	if ($@) {	138	my $row = $rs->next();
131	warn "Something went wrong reading record for biblio $row->biblionumber: $@\n";	139	last if !$row;
132	next;	140	push @biblionumbers, $row->biblionumber;
		141	}
		142	if (@biblionumbers) {
		143	my $marc_records = C4::Biblio::GetMarcBiblios({
		144	biblionumbers => \@biblionumbers,
		145	embed_items => 1
		146	});
		147	while (my ($biblionumber, $marc_record) = each %{$marc_records}) {
		148	my $next = __PACKAGE__->new($marc_record, $biblionumber);
		149	push @marc_records_batch, $next;
		150	}
		151	}
133	}	152	}
134	return $next;	153	return pop @marc_records_batch;
135	}	154	};
136	};	155	}
		156	else {
		157	$next_func = sub {
		158	# Warn and skip bad records, otherwise we break the loop
		159	while (1) {
		160	my $row = $rs->next();
		161	return if !$row;
		162	my $marc = C4::Biblio::GetMarcBiblio({
		163	biblionumber => $row->biblionumber,
		164	embed_items => 1
		165	});
		166	my $next = eval {
		167	__PACKAGE__->new($marc, $row->biblionumber);
		168	};
		169	if ($@) {
		170	warn "Something went wrong reading record for biblio $row->biblionumber: $@\n";
		171	next;
		172	}
		173	return $next;
		174	}
		175	};
		176	}
137	return Koha::MetadataIterator->new($next_func);	177	return Koha::MetadataIterator->new($next_func);
138	}	178	}
139		179