From 1d6516038799121fc344b1ea98a6839eb2a4b590 Mon Sep 17 00:00:00 2001 From: wajasu Date: Wed, 15 Aug 2012 12:51:28 -0500 Subject: [PATCH] Bug 8378 - syntax not working on overdues anymore Content-Type: text/plain; charset="utf-8" Bug 8378 - syntax broken NFC and charset utf8 NFC normalize enqueued letters and add content-type charset=utf-8 This prevents utf8 codes from causing mysql to truncate the 'content' from the point of certain codes, when stored in the message_queue table. This was happenning with the currency symbol generated by Locale::Currency:Format currency_format routine. NFC normalization was only done on the attachment content with its content-type containing "text", as in text/plain. For emails AND attachments, the charset="utf-8" was added to the content-type so mail clients would correctly iterate the utf8 codes, thus preventing mobijake. Signed-off-by: Kyle M Hall Ran through test plan before and after applying patch. Verified that fine syntax does not work pre-patch and does work post-patch for both direct emails and emails to the KohaAdminEmailAddress. --- C4/Letters.pm | 97 +++++++++++++++++++++++++++++++++++++- misc/cronjobs/overdue_notices.pl | 17 +++++-- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/C4/Letters.pm b/C4/Letters.pm index b17cb08..0b6a9be 100644 --- a/C4/Letters.pm +++ b/C4/Letters.pm @@ -31,6 +31,7 @@ use C4::SMS; use C4::Debug; use Date::Calc qw( Add_Delta_Days ); use Encode; +use Unicode::Normalize; use Carp; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); @@ -116,6 +117,33 @@ sub GetLetters { return \%letters; } +=head2 GetLetter( %params ) + + retrieves the letter template + + %params hash: + module => letter module, mandatory + letter_code => letter code, mandatory + branchcode => for letter selection, if missing default system letter taken + Return value: + letter fields hashref (title & content useful) + +=cut + +sub GetLetter { + my %params = @_; + + my $module = $params{module} or croak "No module"; + my $letter_code = $params{letter_code} or croak "No letter_code"; + my $branchcode = $params{branchcode} || ''; + + my $letter = getletter( $module, $letter_code, $branchcode ) + or warn( "No $module $letter_code letter"), + return; + + return $letter; +} + my %letter; sub getletter { my ( $module, $code, $branchcode ) = @_; @@ -409,6 +437,8 @@ sub SendAlerts { =head2 GetPreparedLetter( %params ) + retrieves letter template and performs substituion processing + %params hash: module => letter module, mandatory letter_code => letter code, mandatory @@ -436,14 +466,65 @@ sub GetPreparedLetter { my $module = $params{module} or croak "No module"; my $letter_code = $params{letter_code} or croak "No letter_code"; my $branchcode = $params{branchcode} || ''; + my $tables = $params{tables}; + my $substitute = $params{substitute}; + my $repeat = $params{repeat}; my $letter = getletter( $module, $letter_code, $branchcode ) or warn( "No $module $letter_code letter"), return; + my $prepared_letter = GetProcessedLetter( + module => $module, + letter_code => $letter_code, + letter => $letter, + branchcode => $branchcode, + tables => $tables, + substitute => $substitute, + repeat => $repeat + ); + + return $prepared_letter; +} + +=head2 GetProcessedLetter( %params ) + + given a letter, with possible pre-processing do standard processing + allows one to perform letter template processing beforehand + + %params hash: + module => letter module, mandatory + letter_code => letter code, mandatory + letter => letter, mandatory + branchcode => for letter selection, if missing default system letter taken + tables => a hashref with table names as keys. Values are either: + - a scalar - primary key value + - an arrayref - primary key values + - a hashref - full record + substitute => custom substitution key/value pairs + repeat => records to be substituted on consecutive lines: + - an arrayref - tries to guess what needs substituting by + taking remaining << >> tokensr; not recommended + - a hashref token => @tables - replaces << >> << >> + subtemplate for each @tables row; table is a hashref as above + want_librarian => boolean, if set to true triggers librarian details + substitution from the userenv + Return value: + letter fields hashref (title & content useful) + +=cut + +sub GetProcessedLetter { + my %params = @_; + + my $module = $params{module} or croak "No module"; + my $letter_code = $params{letter_code} or croak "No letter_code"; + my $letter = $params{letter} or croak "No letter"; + my $branchcode = $params{branchcode} || ''; my $tables = $params{tables}; my $substitute = $params{substitute}; my $repeat = $params{repeat}; + $tables || $substitute || $repeat or carp( "ERROR: nothing to substitute - both 'tables' and 'substitute' are empty" ), return; @@ -682,6 +763,12 @@ sub EnqueueLetter { return; } + # It was found that the some utf8 codes, cause the text to be truncated from that point onward when stored, + # so we normalize utf8 with NFC so that mysql will store 'all' of the content in its TEXT column type + # Note: It is also done in _add_attachments accordingly. + $params->{'letter'}->{'title'} = NFC($params->{'letter'}->{'title'}); # subject + $params->{'letter'}->{'content'} = NFC($params->{'letter'}->{'content'}); + # If we have any attachments we should encode then into the body. if ( $params->{'attachments'} ) { $params->{'letter'} = _add_attachments( @@ -852,11 +939,17 @@ sub _add_attachments { $message->attach( Type => $letter->{'content-type'} || 'TEXT', Data => $letter->{'is_html'} - ? _wrap_html($letter->{'content'}, $letter->{'title'}) - : $letter->{'content'}, + ? _wrap_html($letter->{'content'}, NFC($letter->{'title'})) + : NFC($letter->{'content'}), ); foreach my $attachment ( @$attachments ) { + + if ($attachment->{'content'} =~ m/text/o) { # NFC normailze any "text" related content-type attachments + $attachment->{'content'} = NFC($attachment->{'content'}); + } + $attachment->{'filename'} = NFC($attachment->{'filename'}); + $message->attach( Type => $attachment->{'type'}, Data => $attachment->{'content'}, diff --git a/misc/cronjobs/overdue_notices.pl b/misc/cronjobs/overdue_notices.pl index 1243add..e83bd39 100755 --- a/misc/cronjobs/overdue_notices.pl +++ b/misc/cronjobs/overdue_notices.pl @@ -432,6 +432,12 @@ END_SQL next PERIOD; } + my $letter_template = C4::Letters::GetLetter ( + module => 'circulation', + letter_code => $overdue_rules->{"letter$i"}, + branchcode => $branchcode + ); + # $letter->{'content'} is the text of the mail that is sent. # this text contains fields that are replaced by their value. Those fields must be written between brackets # The following fields are available : @@ -503,6 +509,7 @@ END_SQL my $letter = parse_letter( { letter_code => $overdue_rules->{"letter$i"}, + letter => $letter_template, borrowernumber => $borrowernumber, branchcode => $branchcode, items => \@items, @@ -602,11 +609,12 @@ END_SQL my $attachment = { filename => defined $csvfilename ? 'attachment.csv' : 'attachment.txt', - type => 'text/plain', + type => 'text/plain; charset="utf-8"', content => $content, }; my $letter = { + 'content-type' => 'text/plain; charset="utf-8"', title => 'Overdue Notices', content => 'These messages were not sent directly to the patrons.', }; @@ -671,7 +679,7 @@ sub parse_letter { my $currency_format; if ($params->{'letter'}->{'content'} =~ m/(.*)<\/fine>/o) { # process any fine tags... $currency_format = $1; - $params->{'letter'}->{'content'} =~ s/.*<\/fine>/<>/o; + $params->{'letter'}->{'content'} =~ s/.*<\/fine>/<>/o; } my @item_tables; @@ -696,13 +704,14 @@ sub parse_letter { } } - return C4::Letters::GetPreparedLetter ( + return C4::Letters::GetProcessedLetter ( module => 'circulation', letter_code => $params->{'letter_code'}, + letter => $params->{'letter'}, branchcode => $params->{'branchcode'}, tables => \%tables, substitute => $substitute, - repeat => { item => \@item_tables }, + repeat => { item => \@item_tables } ); } -- 1.7.2.5