From d20b75c5365cd390e415faa0e35af339c6aac1be Mon Sep 17 00:00:00 2001 From: Mason James Date: Thu, 28 Apr 2016 23:09:45 +1200 Subject: [PATCH] Bug 10756 - Carousel Display of New Titles on OPAC home page test plan... 1/ import some bibs into your database for testing (important) $ wget http://www.loc.gov/catdir/cpso/RDAtest/extra_bib_marc.zip $ unzip extra_bib_marc.zip $ bulkmarcimport.pl -b -m ISO2709 -file ./formal_extra_all_final.bib 2/ apply patches 3/ execute atomic update to add syspref and 'Carousel' table $ perl ./installer/data/mysql/atomicupdate/Bug-10756-carousel.pl 4/ open OPAC homepage in web-browser, observe no change 5/ set 'OpacCarousel' syspref to 'display' 6/ open OPAC homepage in web-browser, observe a new carousel :0) extra QA tests... 7/ run 5 carousel tests $ prove ./t/db_dependent/Carousel.t 8/ run qa-tool on patchset $ qa -c 2 Signed-off-by: Magnus Enger Followed the test plan, everything worked as expected. The front page displays a nice coverflow widget. More parameters to control the widget would be nice, but that could be added later. On first running this I got a nasty 500 error: "Can't call method "is_valid" on an undefined value at /home/magnus/scripts/kohaclone/Koha/Carousel.pm line 128." But I think I found a cure for that and will propose a followup patch. --- Koha/Carousel.pm | 278 +++++++++++++++++++++ Koha/Schema/Result/Carousel.pm | 79 ++++++ debian/koha-common.cron.daily | 2 +- .../data/mysql/atomicupdate/Bug-10756-carousel.pl | 20 ++ .../prog/en/modules/admin/preferences/opac.pref | 7 + .../bootstrap/en/includes/opac-bottom.inc | 16 ++ .../opac-tmpl/bootstrap/en/modules/opac-main.tt | 18 ++ misc/cronjobs/cleanup_database.pl | 27 +- opac/opac-main.pl | 9 + t/db_dependent/Carousel.t | 243 ++++++++++++++++++ 10 files changed, 696 insertions(+), 3 deletions(-) create mode 100755 Koha/Carousel.pm create mode 100644 Koha/Schema/Result/Carousel.pm create mode 100755 installer/data/mysql/atomicupdate/Bug-10756-carousel.pl create mode 100755 t/db_dependent/Carousel.t diff --git a/Koha/Carousel.pm b/Koha/Carousel.pm new file mode 100755 index 0000000..ec6a7f7 --- /dev/null +++ b/Koha/Carousel.pm @@ -0,0 +1,278 @@ +package Koha::Carousel; + +# Copyright 2016 Mason James +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use strict; +use warnings; +use C4::Koha; +use C4::Biblio; +use C4::Debug; + +use LWP::Simple; +use List::Util qw(shuffle ); +use Business::ISBN; + +if ( $ENV{DEBUG} ) { + use Time::HiRes qw/gettimeofday tv_interval/; + use Carp; +} + +use vars qw($VERSION @ISA @EXPORT); + +BEGIN { + $VERSION = 1.00; + + require Exporter; + @ISA = qw( Exporter ); + + push @EXPORT, qw( + &GetRecentBibs + ); +} + +=head1 NAME + +Koha::Carousel - A module to handle subroutines for the 'new items' carousel + +=head1 DESCRIPTION + +This module has 1 subroutine, + + GetRecentBibs() returns a hashref of bibs with a matching amazon bookcover url + +=head1 SYNOPSIS + + use Koha::Carousel + + # give me a hashref of 10 random new bibs + my $new_bibs_href = GetRecentBibs(); + + # give me a hashref of 20 random new bibs + my $new_bibs_href = GetRecentBibs('20'); + +=cut + +sub GetRecentBibs { + + my $caro_num = shift; + + # speed stats for debugging. + my $tt0; + my $tt1; + if ( $ENV{DEBUG} ) { + $tt0 = [gettimeofday]; + } + + # set vars = 0, to shush warnings + my ( $amz_hit_cnt, $amz_miss_cnt, $dupes, $cache_hits, $fetches, $hits, + $cache_misses ) + = (0) x 7; + my ( @bibs_stage1, @bibs_stage2, @bibs_stage3 ); + + my $ua = new LWP::UserAgent( 'max_redirect' => 0 ); + + # 3 different limits, for narrowing the list of bib results + my $stage1_limit = 300; + my $stage2_limit = 150; + my $stage3_limit = $caro_num ? $caro_num : 10; + +# Initial SQL query to get recently added bibs +# we choose a 'reasonable' limit here, to ensure we don't make our dataset too small + + my $bib_rs = Koha::Database->new()->schema->resultset('Biblio')->search( + { 'biblioitems.isbn' => { '!=', undef }, }, + { + join => 'biblioitems', + '+select' => ['biblioitems.isbn'], + '+as' => ['isbn'], + + rows => $stage1_limit, + desc => 'datecreated', + } + ); + + # run a loop to get $stage1_limit bibs with checksum verified ISBNs. + my $i; + while ( my $r1 = $bib_rs->next ) { + + # test checksum of ISBN10 + my $isbn_tmp = $r1->get_column('isbn'); + my $isbn; + + my @ibs = split( /\|/, $isbn_tmp ); + foreach my $ii (@ibs) { + + # strip trailing (comments) from isbn + $ii =~ s/\(.*$//g; + + my $isbn_obj = Business::ISBN->new($ii); + next unless $isbn_obj; + + my $isbn10; + $isbn10 = $isbn_obj->as_isbn10; + if ( $isbn10->is_valid() ) { + $isbn = $isbn10->as_string( [] ); + last; + } + } + + next unless $isbn; + + my $bib = { + biblionumber => $r1->biblionumber, + title => $r1->title, + author => $r1->author, + isbn => $isbn, + }; + + push @bibs_stage1, $bib; + + } # end while + + # run a loop to remove any dupes + $i = 0; + foreach my $bib (@bibs_stage1) { + + # check for dups before adding + my $dupe = 0; + foreach my $b (@bibs_stage2) { + if ( $bib->{isbn} eq $b->{isbn} ) { + $dupe = 1; + $dupes++; + last; + } + } + next if $dupe == 1; + push @bibs_stage2, $bib; + last if ++$i == $stage2_limit; + } # end while + + #randomise our bibs + @bibs_stage2 = shuffle @bibs_stage2; + + # loop, until we get enough cover images + # add each amazon lookup attempt to the carousel db table, for future caching + + for my $bib (@bibs_stage2) { + + # check for an existing ISBN row, in caro table + my $caro_rs = + Koha::Database->new()->schema->resultset('Carousel') + ->search( { 'isbn' => { '=', $bib->{'isbn'} }, }, { rows => '1' } ); + + my $cnt = $caro_rs->count; + + my $rs = $caro_rs->next; + my $amz_miss; + my $image_url; + + # if not db match, do amazon lookup.... + if ( $cnt == 0 ) { + + # no row in table, so fetch from amazon + $image_url = + 'http://images.amazon.com/images/P/' + . $bib->{isbn} + . '.01._THUMBZZZ.jpg'; + + my $req = HTTP::Request->new( 'GET', $image_url ); + my $res = $ua->request($req); + my $res_size = $res->headers->{'content-length'}; + + $fetches++; + + # warn $res_size; + # 43 bytes returned, means lookup was a miss :'( + $amz_miss = 1 if ( $res_size == 43 ); + + # insert lookup result into table + $caro_rs->create( + { + isbn => $bib->{'isbn'}, + image_url => ( $amz_miss ? undef : $image_url ) + } + ); + + if ($amz_miss) { + $amz_miss_cnt++; + next; + } + else { + # else, got a match from amazon + $amz_hit_cnt++; + } + + } # end + # else, got result from db.. + else { + + # but no image, so skip :/ + unless ( $rs->get_column('image_url') ) { + $cache_misses++; + next; + } + else { + # db result has image, so use ;) + $image_url = $rs->get_column('image_url'); + $cache_hits++; + } + } # end for + + # remove glitchy trailing chars from title/author for display /:, + foreach ( $bib->{title}, $bib->{author} ) { + next unless $_; # shush warns + s|/$||; + s|:$||; + s|,$||; + s/[ \t]+$//g; + } + + $bib->{image_url} = $image_url; + $bib->{id} = $hits; + + push @bibs_stage3, $bib; + + $hits++; + last if $hits == $stage3_limit; + } + + # if debug, print some extra stats + if ( $ENV{DEBUG} ) { + $tt1 = [gettimeofday]; + carp 'carousel: load time ' . tv_interval( $tt0, $tt1 ); + carp "carousel: db_hits: $cache_hits, amz_fetches:$fetches"; + carp +"amz_hit:$amz_hit_cnt, amz_miss:$amz_miss_cnt, dupes:$dupes, hits: $hits, misses: $cache_misses, needs: $stage3_limit"; + } + + return \@bibs_stage3; +} + +=head1 EXPORT + +None by default. + +=head1 AUTHOR + +Mason James, Emason@calyx.net.auE + +=cut + +1; + +__END__ diff --git a/Koha/Schema/Result/Carousel.pm b/Koha/Schema/Result/Carousel.pm new file mode 100644 index 0000000..3cd2581 --- /dev/null +++ b/Koha/Schema/Result/Carousel.pm @@ -0,0 +1,79 @@ +use utf8; +package Koha::Schema::Result::Carousel; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Koha::Schema::Result::Carousel + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("carousel"); + +=head1 ACCESSORS + +=head2 isbn + + data_type: 'varchar' + is_nullable: 0 + size: 30 + +=head2 image_url + + data_type: 'varchar' + is_nullable: 1 + size: 255 + +=head2 timestamp + + data_type: 'timestamp' + datetime_undef_if_invalid: 1 + default_value: current_timestamp + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "isbn", + { data_type => "varchar", is_nullable => 0, size => 255 }, + "image_url", + { data_type => "varchar", is_nullable => 0, size => 255 }, + "timestamp", + { + data_type => "timestamp", + datetime_undef_if_invalid => 1, + default_value => \"current_timestamp", + is_nullable => 0, + }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("isbn"); + + +# Created by DBIx::Class::Schema::Loader v0.07025 @ 2014-11-15 02:47:15 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:WlpGsPHQes9PIDaN4/6zaA + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/debian/koha-common.cron.daily b/debian/koha-common.cron.daily index cbd7eed..9830840 100644 --- a/debian/koha-common.cron.daily +++ b/debian/koha-common.cron.daily @@ -21,7 +21,7 @@ koha-foreach --enabled --email /usr/share/koha/bin/cronjobs/advance_notices.pl - koha-foreach --enabled /usr/share/koha/bin/cronjobs/membership_expiry.pl -c koha-foreach --enabled /usr/share/koha/bin/cronjobs/holds/cancel_expired_holds.pl >/dev/null 2>&1 koha-foreach --enabled /usr/share/koha/bin/cronjobs/services_throttle.pl > /dev/null 2>&1 -koha-foreach --enabled /usr/share/koha/bin/cronjobs/cleanup_database.pl --sessions --zebraqueue 10 --list-invites +koha-foreach --enabled /usr/share/koha/bin/cronjobs/cleanup_database.pl --sessions --zebraqueue 10 --list-invites --carousel koha-foreach --enabled --noemail /usr/share/koha/bin/cronjobs/cleanup_database.pl --mail koha-foreach --enabled /usr/share/koha/bin/cronjobs/holds/auto_unsuspend_holds.pl > /dev/null 2>&1 koha-foreach --enabled /usr/share/koha/bin/cronjobs/automatic_renewals.pl diff --git a/installer/data/mysql/atomicupdate/Bug-10756-carousel.pl b/installer/data/mysql/atomicupdate/Bug-10756-carousel.pl new file mode 100755 index 0000000..724f9f0 --- /dev/null +++ b/installer/data/mysql/atomicupdate/Bug-10756-carousel.pl @@ -0,0 +1,20 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use C4::Context; +my $dbh = C4::Context->dbh; + +$dbh->do( +q| +CREATE TABLE `carousel` ( + `isbn` varchar(255) NOT NULL, + `image_url` varchar(255) NULL, + `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`isbn`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 | +); + + $dbh->do("INSERT INTO `systempreferences` (variable,value,explanation,options,type) VALUES('OpacCarousel','0','Display the new-items carousel on OPAC home page.','','YesNo')"); + +print "Upgrade done (Bug 10756 - Carousel Display of New Titles on OPAC)\n"; diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/opac.pref b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/opac.pref index f092767..9f85f19 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/opac.pref +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/admin/preferences/opac.pref @@ -490,6 +490,13 @@ OPAC: no: Don't display - the acquisition details on OPAC detail pages. - + - pref: OPACCarousel + default: 0 + choices: + yes: Display + no: Don't display + - the 'new items' carousel on OPAC home page. + - - "Use the following as the OPAC ISBD template:" - pref: OPACISBD type: textarea diff --git a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc b/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc index 9e7a15c..7c14881 100644 --- a/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc +++ b/koha-tmpl/opac-tmpl/bootstrap/en/includes/opac-bottom.inc @@ -189,6 +189,22 @@ $.widget.bridge('uitooltip', $.ui.tooltip); [% END %] +[% IF OpacCarousel %] + + +[% END %] + [% IF OPACLocalCoverImages %]