From f4aec0aaeeb7dbc5f2b24056024d2ea33c704e70 Mon Sep 17 00:00:00 2001 From: David Cook Date: Thu, 4 Jun 2020 12:34:19 +1000 Subject: [PATCH] Create experimental support for RabbitMQ-based background tasks This patch includes a koha-mq-scheduler service which creates a mq_scheduler.pl daemon, which listens to RabbitMQ queues. This patch also includes an audit-authorities.pl web script, which sends a message to RabbitMQ. The mq_scheduler.pl daemon loads Koha::Prosentient::MQ::AuditAuthorities.pm, and runs a class method which does a background task to audit bib-auth linkages. Koha community work: https://bugs.koha-community.org/bugzilla3//show_bug.cgi?id=22417 --- Koha/Prosentient/MQ.pm | 79 ++++++ Koha/Prosentient/MQ/AuditAuthorities.pm | 287 +++++++++++++++++++++ etc/prosentient/koha-mq-scheduler@.service | 18 ++ .../modules/tools/prosentient/audit-authorities.tt | 84 ++++++ misc/prosentient/mq_scheduler.pl | 97 +++++++ tools/prosentient/audit-authorities.pl | 106 ++++++++ 6 files changed, 671 insertions(+) create mode 100644 Koha/Prosentient/MQ.pm create mode 100644 Koha/Prosentient/MQ/AuditAuthorities.pm create mode 100644 etc/prosentient/koha-mq-scheduler@.service create mode 100644 koha-tmpl/intranet-tmpl/prog/en/modules/tools/prosentient/audit-authorities.tt create mode 100755 misc/prosentient/mq_scheduler.pl create mode 100755 tools/prosentient/audit-authorities.pl diff --git a/Koha/Prosentient/MQ.pm b/Koha/Prosentient/MQ.pm new file mode 100644 index 0000000000..bfcd197039 --- /dev/null +++ b/Koha/Prosentient/MQ.pm @@ -0,0 +1,79 @@ +package Koha::Prosentient::MQ; + +use Modern::Perl; +use Module::Load::Conditional qw/check_install/; +use C4::Context; + +sub _get_config { + my $config; + my $context = C4::Context->new(); + if ($context && $context->{prosentient} && $context->{prosentient}->{message_broker}){ + $config = $context->{prosentient}->{message_broker}; + } + return $config; +} + +sub is_supported { + my $supported = 0; + my $stomp_lib = check_install( module => 'Net::Stomp' ); + my $config = _get_config(); + if ($stomp_lib && $config){ + $supported = 1; + } + return $supported; +} + +sub queue { + my ($queue_id) = @_; + my $queue; + if ( is_supported() ){ + my $config = _get_config(); + if ($config && $config->{queue} && $queue_id){ + my $queue_config = $config->{queue}->{$queue_id}; + if ($queue_config && $queue_config->{dest}){ + $queue = $queue_config->{dest}; + } + } + } + return $queue; +} + +sub queues { + my @queues = (); + if ( is_supported() ){ + my $config = _get_config(); + if ($config){ + my $queue_hash = $config->{queue}; + foreach my $queue_id (keys %$queue_hash){ + my $queue_config = $queue_hash->{$queue_id}; + if ($queue_config && $queue_config->{dest}){ + my $queue = { + id => $queue_id, + dest => $queue_config->{dest}, + }; + push(@queues,$queue); + } + } + } + } + return \@queues; +} + +sub connection { + my $connection; + if ( is_supported() ){ + #NOTE: During the experimental phase, lazy load this library + require Net::Stomp; + my $config = _get_config(); + if ($config){ + my $stomp = Net::Stomp->new( { hostname => $config->{hostname}, port => $config->{port} } ); + my $frame = $stomp->connect( { login => $config->{user}, passcode => $config->{password}, host => $config->{vhost} } ); + if ($frame && $frame->command eq 'CONNECTED'){ + $connection = $stomp; + } + } + } + return $connection; +} + +1; diff --git a/Koha/Prosentient/MQ/AuditAuthorities.pm b/Koha/Prosentient/MQ/AuditAuthorities.pm new file mode 100644 index 0000000000..0957bb0ab8 --- /dev/null +++ b/Koha/Prosentient/MQ/AuditAuthorities.pm @@ -0,0 +1,287 @@ +package Koha::Prosentient::MQ::AuditAuthorities; + +use Modern::Perl; +use Template; +use Mail::Sendmail; +use Email::Valid; +use Archive::Zip qw( :ERROR_CODES :CONSTANTS ); +use Encode; +use MIME::QuotedPrint; +use MIME::Base64; + +use C4::Context; +use Koha::Database; +use Koha::MarcSubfieldStructures; +use Koha::Biblios; + +my $email_template = < + + + + + + + + + + + + + + + + [% FOREACH result IN results %] + + + + + + + + + + + [% END %] +
BiblionumberTagAuth typeBib headingLinkage in bibDescriptionMatched authority idMatched authority heading
[% result.biblionumber %][% result.field %]$[% result.subfield %][% result.authtypecode %][% result.bib_full %][% result.linkage %][% result.desc %][% result.matched_id %][% result.matched_heading %]
+ + +EMAIL_TEMPLATE + +sub do_task { + my ($class,$args) = @_; + my $completed = 1; + my $debug = $args->{debug} // 0; + my $field = $args->{field}; + my $subfield = $args->{subfield}; + my $authtypecode = $args->{authtypecode}; + my $email = $args->{email}; + $debug && warn __PACKAGE__ . " " . "starting task"; + if ($field && $subfield && $authtypecode){ + my $results = _get_results({ + field => $field, + subfield => $subfield, + authtypecode => $authtypecode, + }); + if (@$results){ + $debug && warn __PACKAGE__ . " " . scalar @$results . " results"; + my $report = _create_report({ + results => $results, + }); + if ($report){ + $debug && warn __PACKAGE__ . " " . "report created"; + my $sent = _email_report({ + email => $email, + report => $report, + tag => $field . '$' . $subfield, + }); + if ($sent){ + $debug && warn __PACKAGE__ . " " . "email sent"; + } + } + } + } + $debug && warn __PACKAGE__ . " " . "completed task"; + return $completed; +} + +sub _email_report { + my ($args) = @_; + my $sent; + my $email = $args->{email}; + my $report = $args->{report}; + my $tag = $args->{tag}; + if ($email && $report && $tag){ + if ( Email::Valid->address($email) ){ + my $zip = Archive::Zip->new(); + $zip->addString(Encode::encode("UTF-8",$report),"authorityreport_$tag.html"); + my $memory_file = ''; #scalar as a file + open(my $fh, '>', \$memory_file); + my $status = $zip->writeToFileHandle($fh); + if ($status == AZ_OK){ + $fh->close; + my $zipfile = encode_base64($memory_file); + + my %mail = (); + $mail{from} = 'ADMIN@DOMAIN.com'; + $mail{to} = $email; + $mail{subject} = Encode::encode("UTF-8","Authorities audit report for your catalogue on tag $tag"); + my $boundary = "====" . time() . "===="; + $mail{'content-type'} = "multipart/mixed; boundary=\"$boundary\""; + $boundary = '--' . $boundary; + my $body = encode_qp(Encode::encode("UTF-8","Please find attached your authorities audit report for review.")); + $mail{body} = <{field}; + my $subfield = $args->{subfield}; + my $authtypecode = $args->{authtypecode}; + if ($field && $subfield && $authtypecode){ + + my $schema = Koha::Database->new->schema; + my $authtype = $schema->resultset('AuthType')->search({ + authtypecode => $authtypecode, + })->first; + + my $auth_tag = $authtype->auth_tag_to_report; + + my %auth_lookup_by_id = (); + my %auth_lookup_by_full = (); + my %auth_lookup_by_primary = (); + my $dbh = C4::Context->dbh; + my $sql = qq~ + SELECT + authid, + ExtractValue(marcxml,'//datafield[\@tag="$auth_tag"]/*') as full_header, + ExtractValue(marcxml,'//datafield[\@tag="$auth_tag"]/subfield[\@code="a"]') as primary_header + FROM auth_header + WHERE authtypecode = ? + ~; + my $sth = $dbh->prepare($sql); + $sth->execute($authtypecode); + while (my $row = $sth->fetchrow_hashref){ + $auth_lookup_by_id{ $row->{authid} } = { full => $row->{full_header}, primary => $row->{primary_header}, }; + if ( ! $auth_lookup_by_full{ $row->{full_header} } ){ + $auth_lookup_by_full{ $row->{full_header} } = []; + } + push(@{$auth_lookup_by_full{ $row->{full_header} }},$row->{authid}); + + if ( ! $auth_lookup_by_primary{ $row->{primary_header} } ){ + $auth_lookup_by_primary{ $row->{primary_header} }= []; + } + push(@{$auth_lookup_by_primary{ $row->{primary_header} }},$row->{authid}); + } + + + my @frameworks_to_audit = Koha::MarcSubfieldStructures->search( + { + tagfield => $field, + tagsubfield => $subfield, + authtypecode => $authtypecode, + } + ); + my @search_conditions = (); + foreach my $structure (@frameworks_to_audit){ + my $frameworkcode = $structure->frameworkcode; + my $condition = { frameworkcode => $frameworkcode }; + push(@search_conditions,$condition); + } + + + + my @biblios = Koha::Biblios->search(\@search_conditions); + foreach my $biblio (@biblios){ + my $record = $biblio->metadata->record; + if ($record){ + my @marc_fields = $record->field($field); + foreach my $marc_field (@marc_fields){ + my $result = { + biblionumber => $biblio->biblionumber, + field => $field, + subfield => $subfield, + bib_full => '', + authtypecode => $authtypecode, + linkage => '', + matched_heading => '', + matched_id => '', + desc => 'Unlinked and no match', + }; + #Check if it already has a $9 linkage + my $bib_field = $marc_field->clone(); + my $linkage = $bib_field->subfield('9'); + $bib_field->delete_subfield('9'); + my $bib_subfield = $bib_field->subfield($subfield); + my $bib_full = $bib_field->as_string; + $result->{bib_full} = $bib_full; + if ( $linkage){ + my $linked_auth = $auth_lookup_by_id{ $linkage }; + if ($linked_auth){ + $result->{linkage} = $linkage; + if ( $linked_auth->{full} eq $bib_field->as_string){ + $result->{desc} = "Linked heading 100% match"; + $result->{matched_heading} = $linked_auth->{full}; + } elsif ( $bib_subfield && $linked_auth->{primary} eq $bib_subfield ){ + $result->{desc} = "Linked heading partial match"; + $result->{matched_heading} = $linked_auth->{primary}; + + } else { + $result->{desc} = "Linked heading does not match"; + $result->{matched_heading} = $linked_auth->{full}; + } + } else { + $result->{desc} = "Linked but unable to find authority record"; + } + } else { + my $full_match = $auth_lookup_by_full{ $bib_full }; + if ($full_match) { + $result->{desc} = "Unlinked but found 100% match"; + $result->{matched_id} = join(',',@$full_match); + $result->{matched__heading} = $bib_full; + } else { + if ($bib_subfield){ + my $primary_match = $auth_lookup_by_primary{ $bib_subfield }; + if ($primary_match){ + $result->{desc} = "Unlinked but found partial match"; + $result->{matched_id} = join(',',@$primary_match); + $result->{matched_heading} = $bib_subfield; + } + } + } + } + push(@results,$result); + } + } + } + } + return \@results; +} + +1; + +sub _get_template { + my $template; + while (my $line = ){ + $template .= $line; + } + return $template; +} + +sub _create_report { + my ($args) = @_; + my $report; + my $results = $args->{results}; + if ($results){ + my $template_engine = Template->new({ ENCODING => 'UTF-8', }); + my $template = $email_template; + $template_engine->process(\$template, { results => $results, }, \$report); + } + return $report; +} + diff --git a/etc/prosentient/koha-mq-scheduler@.service b/etc/prosentient/koha-mq-scheduler@.service new file mode 100644 index 0000000000..3c21e46a1b --- /dev/null +++ b/etc/prosentient/koha-mq-scheduler@.service @@ -0,0 +1,18 @@ +# koha-mq-scheduler@.service +# /etc/systemd/system/koha-mq-scheduler@.service +[Unit] +Description=Koha Message Queue Work Scheduler +After=syslog.target network.target rabbitmq-server.service + +[Service] +Environment=PERL5LIB=/KOHA/%i/lib +Environment=KOHA_CONF=/KOHA/%i/etc/koha-conf.xml +ExecStart=/KOHA/%i/bin/prosentient/mq_scheduler.pl -v +Restart=on-failure +RestartSec=5s +StartLimitBurst=3 +StartLimitInterval=60 +SyslogIdentifier=koha-mq-scheduler_%i + +[Install] +WantedBy=multi-user.target diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/tools/prosentient/audit-authorities.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/prosentient/audit-authorities.tt new file mode 100644 index 0000000000..5d745318e4 --- /dev/null +++ b/koha-tmpl/intranet-tmpl/prog/en/modules/tools/prosentient/audit-authorities.tt @@ -0,0 +1,84 @@ +[% USE raw %] +[% USE Asset %] +[% SET footerjs = 1 %] + [% INCLUDE 'doc-head-open.inc' %] + Koha › Tools › Authorities audit + [% INCLUDE 'doc-head-close.inc' %] + + + +[% INCLUDE 'header.inc' %] +[% INCLUDE 'cat-search.inc' %] + + + +
+
+
+
+

Authorities audit

+ [% IF ( tags_to_audit ) %] +
+
+ Select tag to audit +
    +
  1. + + +
    2. +
+
+ +
+ [% ELSIF ( audit ) %] + [% IF ( message_enqueued ) %] +

The authorities audit may take a long time to run depending on the size of your authority file.

+

Your audit has been scheduled to run offline in batch mode. Please do not refresh or resubmit.

+ [% IF ( email ) %] +

When the audit is complete, a report will be emailed to "[% email %]".

+ [% END %] + [% ELSE %] +

Unable to schedule your authorities audit. Please contact your system administrator for assistance

+ [% END %] +
+ Audit details +
    +
  1. + + [% field %] +
    2. +
  2. + + [% subfield %] +
    3. +
  3. + + [% authtypecode %] +
    4. +
+
+ + [% END %] +
+
+ +
+ +
+
+ +[% MACRO jsinclude BLOCK %] +[% END %] + +[% INCLUDE 'intranet-bottom.inc' %] diff --git a/misc/prosentient/mq_scheduler.pl b/misc/prosentient/mq_scheduler.pl new file mode 100755 index 0000000000..1d26308ca0 --- /dev/null +++ b/misc/prosentient/mq_scheduler.pl @@ -0,0 +1,97 @@ +#!/usr/bin/perl +# +# Copyright Prosentient Systems 2020 +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# Koha is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +use Modern::Perl; +use Getopt::Long; +use Pod::Usage; +use JSON; +use Module::Load::Conditional qw/can_load/; + +use Koha::Prosentient::MQ; + +=head1 SYNOPSIS + + pro_mq_worker.pl [-h|--help] [--man] + +=cut + +#Make STDOUT hot so STDOUT logging doesn't suffer from buffering +$|++; + +my ($help,$man,$verbose); + +GetOptions( + '-h|help' => \$help, + '--man' => \$man, + '-v' => \$verbose, +) || pod2usage(2); + +pod2usage(1) if $help; +pod2usage( -verbose => 2 ) if $man; + +if ( Koha::Prosentient::MQ::is_supported() ){ + my $conn = Koha::Prosentient::MQ::connection(); + if ($conn){ + my $queues = Koha::Prosentient::MQ::queues(); + if (@$queues){ + foreach my $queue (@$queues){ + $conn->subscribe({ + destination => $queue->{dest}, + 'ack' => 'client', + 'activemq.prefetchSize' => 1, + }); + } + } + MSG: while (1){ + my $frame = $conn->receive_frame; + if (! defined $frame){ + next MSG; + } + $verbose && print "Message received\n"; + my $body = $frame->body; + if ($body){ + my $task = decode_json($body); + if ( $task && $task->{class} && $task->{method} ){ + my $class = $task->{class}; + my $method = $task->{method}; + my $args = $task->{args}; + if ( can_load( modules => { $class => undef, } ) ){ + my $completed = $class->$method($args); + if ($completed){ + $conn->ack( { frame => $frame } ); + $verbose && print "Message processed and task completed\n"; + next MSG; + } + } else { + warn "Unable to load $class"; + } + } + } + #FIXME: This should be a 'nack' instead of an 'ack', + #but we don't want the scheduler stuck in a loop trying + #to process the same message over and over again, if it fails to 'ack' + $conn->ack( { frame => $frame } ); + $verbose && print "Task failed but message acknowledged\n"; + } + } +} else { + warn "Prosentient Koha Messsage Queue not supported!"; + exit 1; +} + diff --git a/tools/prosentient/audit-authorities.pl b/tools/prosentient/audit-authorities.pl new file mode 100755 index 0000000000..69868f04ee --- /dev/null +++ b/tools/prosentient/audit-authorities.pl @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +# Copyright 2020 Prosentient Systems +# +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; +use CGI qw ( -utf8 ); +use JSON; + +use C4::Auth; +use C4::Koha; +use C4::Context; +use C4::Output; + +use Koha::MarcSubfieldStructures; +use Koha::Prosentient::MQ; + +my $cgi = new CGI; +my ( $template, $borrowernumber, $cookie ) = get_template_and_user( + { + template_name => "tools/prosentient/audit-authorities.tt", + query => $cgi, + type => "intranet", + authnotrequired => 0, + flagsrequired => { tools => '*' }, + debug => 1, + } +); +my $tag = $cgi->param('tag'); +if ( $tag && $tag =~ /([0-9a-zA-Z]{3})([0-9a-zA-Z%]{1})(.*)/ ){ + my $field = $1; #field should always be [0-9]{3} but 01e exists + my $subfield = $2; #subfield should always be [0-9a-zA-Z] but % also exists + my $authtypecode = $3; + my $email; + $template->param( audit => 1 ); + $template->param( field => $field ); + $template->param( subfield => $subfield ); + $template->param( authtypecode => $authtypecode ); + my $logged_in_user = Koha::Patrons->find( $borrowernumber ); + if ($logged_in_user){ + my $user_email = $logged_in_user->first_valid_email_address; + if ( $user_email ){ + $email = $user_email; + $template->param( email => $email ); + } else { + my $branch_email = $logged_in_user->library->branchemail; + if ($branch_email){ + $email = $branch_email; + $template->param( email => $branch_email ); + } + } + } + my $task = { + class => 'Koha::Prosentient::MQ::AuditAuthorities', + method => 'do_task', + args => { + field => $field, + subfield => $subfield, + authtypecode => $authtypecode, + email => $email, + } + }; + my $msg = encode_json($task); + + my $queue = Koha::Prosentient::MQ::queue("auditauthorities"); + if ($queue){ + my $conn = Koha::Prosentient::MQ::connection(); + if ($conn){ + my $sent = $conn->send_with_receipt({ + destination => $queue, + body => $msg, + }); + if ($sent){ + $template->param( message_enqueued => 1 ); + } + } + } +} +else { + my @tags_to_audit = Koha::MarcSubfieldStructures->search( + { + authtypecode => { '!=', '' }, + }, + { + group_by => [qw/tagfield tagsubfield authtypecode/], + } + ); + if (@tags_to_audit){ + $template->param( tags_to_audit => \@tags_to_audit ); + } +} +output_html_with_http_headers $cgi, $cookie, $template->output; -- 2.16.4