From 8ec10611aa969dbf909e33ced3b4c64b18a7d040 Mon Sep 17 00:00:00 2001 From: Pedro Amorim Date: Tue, 12 Aug 2025 12:46:30 +0000 Subject: [PATCH] Bug 39345: Add COUNTER 5.1 support This moves the original SUSHI COUNTER logic to a new Koha::ERM::EUsage::COUNTER::5 class that extends Koha::ERM::EUsage::SushiCounter. Adds a new Koha::ERM::EUsage::COUNTER::5_1 class with COUNTER 5.1 specific logic. Keeps common logic in Koha::ERM::EUsage::SushiCounter. This ensures we support both COUNTER 5.0 and COUNTER 5.1, but should hopefully help us better maintain COUNTER 5.1 going forward, without having to worry about COUNTER 5.0, as the code is seperated. This should also better set us up for adding support for possible future COUNTER releases. Test plan, k-t-d: 1) Enable ERMModule 2) Add a new usage data provider at: /cgi-bin/koha/erm/eusage/usage_data_providers/add 3) Add the required credential parameters e.g. Customer ID and Requestor ID 4) Update the 'Report release' to '5.1', no single quote. (If you add anything other than '5', it'll default to '5.1') 5) Visit the list of data providers and click the 'Run now' button for your data provider: /cgi-bin/koha/erm/eusage/usage_data_providers 6) Click the background job link at the top. Verify the background job runs as expected. 7) After the successful harvest, run a custom report at, ensure the data is all correct: /cgi-bin/koha/erm/eusage/reports 8) Visit the data provider at: /cgi-bin/koha/erm/eusage/usage_data_providers/ 9) Click the 'Import logs' tab, download the COUNTER 5.1 counter file. Open the file and verify the data is correct. --- Koha/ERM/EUsage/COUNTER/5.pm | 335 +++++++++++++++++++++++ Koha/ERM/EUsage/COUNTER/5_1.pm | 468 ++++++++++++++++++++++++++++++++ Koha/ERM/EUsage/CounterFile.pm | 62 ++++- Koha/ERM/EUsage/SushiCounter.pm | 314 +++------------------ 4 files changed, 885 insertions(+), 294 deletions(-) create mode 100644 Koha/ERM/EUsage/COUNTER/5.pm create mode 100644 Koha/ERM/EUsage/COUNTER/5_1.pm diff --git a/Koha/ERM/EUsage/COUNTER/5.pm b/Koha/ERM/EUsage/COUNTER/5.pm new file mode 100644 index 00000000000..e4dd98ec591 --- /dev/null +++ b/Koha/ERM/EUsage/COUNTER/5.pm @@ -0,0 +1,335 @@ +package Koha::ERM::EUsage::COUNTER::5; + +# Copyright 2025 Open Fifth + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use base qw(Koha::ERM::EUsage::SushiCounter); + +=head1 NAME + +Koha::ERM::EUsage::COUNTER::5 - Koha COUNTER 5 Object class + +=head1 API + +=head2 Class Methods + +=head3 new + + my $sushi_counter = + Koha::ERM::EUsage::SushiCounter->new( { response => decode_json( $response->decoded_content ) } ); + +=cut + +sub new { + my ($class) = @_; + my $self = {}; + + bless( $self, $class ); +} + +=head3 _COUNTER_report_header + +Return a COUNTER report header +https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html + +=cut + +sub _COUNTER_report_header { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + + my @metric_types_string = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" ); + + my $begin_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Begin_Date" ); + my $end_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "End_Date" ); + + return ( + [ Report_Name => $header->{Report_Name} || "" ], + [ Report_ID => $header->{Report_ID} || "" ], + [ Release => $header->{Release} || "" ], + [ Institution_Name => $header->{Institution_Name} || "" ], + [ Institution_ID => join( "; ", map( $_->{Type} . ":" . $_->{Value}, @{ $header->{Institution_ID} } ) ) || "" ], + [ Metric_Types => join( "; ", split( /\|/, $metric_types_string[0] ) ) || "" ], + [ Report_Filters => join( "; ", map( $_->{Name} . ":" . $_->{Value}, @{ $header->{Report_Filters} } ) ) || "" ], + + #TODO: Report_Attributes may need parsing, test this with a SUSHI response that provides it + [ Report_Attributes => $header->{Report_Attributes} || "" ], + [ + Exceptions => join( + "; ", map( $_->{Code} . ": " . $_->{Message} . " (" . $_->{Data} . ")", @{ $header->{Exceptions} } ) + ) + || "" + ], + [ Reporting_Period => "Begin_Date=" . $begin_date . "; End_Date=" . $end_date ], + [ Created => $header->{Created} || "" ], + [ Created_By => $header->{Created_By} || "" ], + [""] #empty 13th line (COUNTER 5) + ); +} + +=head3 _get_SUSHI_Name_Value + +Returns "Value" of a given "Name" + +=cut + +sub _get_SUSHI_Name_Value { + my ( $self, $item, $name ) = @_; + + my @value = map( $_->{Name} eq $name ? $_->{Value} : (), @{$item} ); + return $value[0]; +} + +=head3 _get_SUSHI_Type_Value + +Returns "Value" of a given "Type" + +=cut + +sub _get_SUSHI_Type_Value { + my ( $self, $item, $type ) = @_; + + my @value = map( $_->{Type} eq $type ? $_->{Value} : (), @{$item} ); + return $value[0]; +} + +=head3 _get_COUNTER_row_usages + +Returns the total and monthly usages for a row + +=cut + +sub _get_COUNTER_row_usages { + my ( $self, $row, $metric_type ) = @_; + + my @usage_months = $self->_get_usage_months( $self->{sushi}->{header} ); + + my @usage_months_fields = (); + my $count_total = 0; + + foreach my $usage_month (@usage_months) { + my $month_is_empty = 1; + + foreach my $performance ( @{ $row->{Performance} } ) { + my $period = $performance->{Period}; + my $period_usage_month = substr( $period->{Begin_Date}, 0, 7 ); + + my $instances = $performance->{Instance}; + my @metric_type_count = + map( $_->{Metric_Type} eq $metric_type ? $_->{Count} : (), + @{$instances} ); + + if ( $period_usage_month eq $usage_month && $metric_type_count[0] ) { + push( @usage_months_fields, $metric_type_count[0] ); + $count_total += $metric_type_count[0]; + $month_is_empty = 0; + } + } + + if ($month_is_empty) { + push( @usage_months_fields, 0 ); + } + } + return ( $count_total, @usage_months_fields ); +} + +=head3 get_SUSHI_metric_types + + Returns all metric types this SUSHI result has statistics for, as an array of strings. + +=cut + +sub get_SUSHI_metric_types { + my ( $self, $report_row ) = @_; + + my @metric_types = (); + + # Grab all metric_types this SUSHI result has statistics for + foreach my $performance ( @{ $report_row->{Performance} } ) { + my @SUSHI_metric_types = + map( $_->{Metric_Type}, @{ $performance->{Instance} } ); + + foreach my $sushi_metric_type (@SUSHI_metric_types) { + push( @metric_types, $sushi_metric_type ) unless grep { $_ eq $sushi_metric_type } @metric_types; + } + } + + return @metric_types; +} + +=head3 _COUNTER_report_body + +Return the COUNTER report body as an array + +=cut + +sub _COUNTER_report_body { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my $body = $self->{sushi}->{body}; + + my @report_body = (); + + foreach my $report_row ( @{$body} ) { + my @metric_types = $self->get_SUSHI_metric_types($report_row); + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( @report_body, $self->_COUNTER_report_row( $report_row, $metric_type ) ); + } + } + + return @report_body; +} + +=head3 _COUNTER_title_report_row + +Return a COUNTER title for the COUNTER titles report body +https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html#column-headings-elements + +=cut + +sub _COUNTER_title_report_row { + my ( $self, $title_row, $metric_type, $total_usage, $monthly_usages ) = @_; + + my $header = $self->{sushi}->{header}; + my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); + + return ( + [ + # Title + $title_row->{Title} || "", + + # Publisher + $title_row->{Publisher} || "", + + # Publisher_ID + $self->_get_SUSHI_Type_Value( $title_row->{Publisher_ID}, "ISNI" ) || "", + + # Platform + $title_row->{Platform} || "", + + # DOI + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "DOI" ) || "", + + # Proprietary_ID + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Proprietary" ) || "", + + # ISBN + grep ( /ISBN/, @{$specific_fields} ) + ? ( $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "ISBN" ) || "" ) + : (), + + # Print_ISSN + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Print_ISSN" ) || "", + + # Online_ISSN + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Online_ISSN" ) || "", + + # URI - FIXME: What goes in URI? + "", + + # YOP + grep ( /YOP/, @{$specific_fields} ) ? ( $title_row->{YOP} || "" ) : (), + + # Access_Type + grep ( /Access_Type/, @{$specific_fields} ) ? ( $title_row->{Access_Type} || "" ) : (), + + # Metric_Type + $metric_type, + + # Report_Period_Total + $total_usage, + + # Monthly usage entries + @{$monthly_usages} + ] + ); +} + +=head3 _COUNTER_titles_report_column_headings + +Return titles report column headings + +=cut + +sub _COUNTER_titles_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my @month_headings = $self->_get_usage_months( $header, 1 ); + my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); + + return ( + [ + "Title", + "Publisher", + "Publisher_ID", + "Platform", + "DOI", + "Proprietary_ID", + grep ( /ISBN/, @{$specific_fields} ) ? ("ISBN") : (), + "Print_ISSN", + "Online_ISSN", + "URI", + + #"Data_Type", #TODO: Only if requested (?) + #"Section_Type", #TODO: Only if requested (?) + grep ( /YOP/, @{$specific_fields} ) ? ("YOP") : (), + grep ( /Access_Type/, @{$specific_fields} ) ? ("Access_Type") : (), + + #"Access_Method", #TODO: Only if requested (?) + "Metric_Type", + "Reporting_Period_Total", + + # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true + @month_headings + ] + ); +} + +=head3 get_report_type_specific_fields + +Returns the specific fields for a given report_type + +=cut + +sub get_report_type_specific_fields { + my ( $self, $report_type ) = @_; + + my %report_type_map = ( + "TR_B1" => [ 'YOP', 'ISBN' ], + "TR_B2" => [ 'YOP', 'ISBN' ], + "TR_B3" => [ 'YOP', 'Access_Type', 'ISBN' ], + "TR_J3" => ['Access_Type'], + "TR_J4" => ['YOP'], + "IR_A1" => [ + 'Authors', 'Publication_Date', 'Article_Version', 'Print_ISSN', 'Online_ISSN', 'Parent_Title', + 'Parent_Authors', 'Parent_Article_Version', 'Parent_DOI', 'Parent_Proprietary_ID', 'Parent_Print_ISSN', + 'Parent_Online_ISSN', 'Parent_URI', 'Access_Type' + ], + ); + + return $report_type_map{$report_type}; + +} + +1; diff --git a/Koha/ERM/EUsage/COUNTER/5_1.pm b/Koha/ERM/EUsage/COUNTER/5_1.pm new file mode 100644 index 00000000000..39842d0c2e5 --- /dev/null +++ b/Koha/ERM/EUsage/COUNTER/5_1.pm @@ -0,0 +1,468 @@ +package Koha::ERM::EUsage::COUNTER::5_1; + +# Copyright 2025 Open Fifth + +# This file is part of Koha. +# +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . + +use Modern::Perl; + +use base qw(Koha::ERM::EUsage::SushiCounter); + +=head1 NAME + +Koha::ERM::EUsage::COUNTER::5 - Koha COUNTER 5.1 Object class + +=head1 API + +=head2 Class Methods + +=head3 new + + my $sushi_counter = + Koha::ERM::EUsage::SushiCounter->new( { response => decode_json( $response->decoded_content ) } ); + +=cut + +sub new { + my ($class) = @_; + my $self = {}; + + bless( $self, $class ); +} + +=head3 _COUNTER_report_header + +Return a COUNTER report header +https://cop5.projectcounter.org/en/5.1/04-reports/03-title-reports.html + +=cut + +sub _COUNTER_report_header { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + + my $metric_types = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" ); + my @metric_types_array = split( /\|/, $metric_types ); + $metric_types = join( "; ", @metric_types_array ) if @metric_types_array; + + my $begin_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Begin_Date" ); + my $end_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "End_Date" ); + + return ( + [ Report_Name => $header->{Report_Name} || "" ], + [ Report_ID => $header->{Report_ID} || "" ], + [ Release => $header->{Release} || "" ], + [ Institution_Name => $header->{Institution_Name} || "" ], + [ Institution_ID => $self->get_mapped_SUSHI_values( 'Type', $header->{Institution_ID} ) || "" ], + [ Metric_Types => $metric_types || "" ], + [ Report_Filters => $self->get_mapped_SUSHI_values( 'Name', $header->{Report_Filters} ) || "" ], + + #TODO: Report_Attributes may need parsing, test this with a SUSHI response that provides it + [ Report_Attributes => $header->{Report_Attributes} || "" ], + [ + Exceptions => join( + "; ", map( $_->{Code} . ": " . $_->{Message} . " (" . $_->{Data} . ")", @{ $header->{Exceptions} } ) + ) + || "" + ], + [ Reporting_Period => "Begin_Date=" . $begin_date . "; End_Date=" . $end_date ], + [ Created => $header->{Created} || "" ], + [ Created_By => $header->{Created_By} || "" ], + [ Registry_Record => $header->{Registry_Record} || "" ], + [""] #empty 14th line (COUNTER 5.1) + ); +} + +=head3 get_counter51_value + + Given a SUSHI response, get the value for the given key in the correct format for the COUNTER report + https://counter5.cambridge.org/r51/sushi-docs/ + +=cut + +sub get_counter51_value { + my ( $self, $item, $key ) = @_; + my $counter_51_value = $item->{$key}; + if ( ref($counter_51_value) eq 'ARRAY' ) { + return join( "; ", @$counter_51_value ); + } else { + return $counter_51_value; + } +} + +=head3 _get_SUSHI_Name_Value + + Given a SUSHI response, get the value for the given key in the correct format for the COUNTER report + https://counter5.cambridge.org/r51/sushi-docs/ + + It will return a string of the form "Name:Value;Name:Value;..." + If the value is an array with a single element, just the value will be returned. + If the value is an array with multiple elements, they will be joined with "|" + +=cut + +sub get_mapped_SUSHI_values { + my ( $self, $key, $sushi_type_value ) = @_; + + return join( + "; ", + map { + $_ . ":" + . ( + ref $sushi_type_value->{$_} eq 'ARRAY' + && scalar( @{ $sushi_type_value->{$_} } ) == 1 ? $sushi_type_value->{$_}[0] + : ( ref $sushi_type_value->{$_} eq 'ARRAY' && scalar( @{ $sushi_type_value->{$_} } ) > 1 ) + ? join( "|", @{ $sushi_type_value->{$_} } ) + : $sushi_type_value->{$_} + ) + } keys %{$sushi_type_value} + ); +} + +=head3 _get_SUSHI_Name_Value + +Returns "Value" of a given "Name" + +=cut + +sub _get_SUSHI_Name_Value { + my ( $self, $item, $name ) = @_; + + return $self->get_counter51_value( $item, $name ); +} + +=head3 _get_SUSHI_Type_Value + +Returns "Value" of a given "Type" + +=cut + +sub _get_SUSHI_Type_Value { + my ( $self, $item, $type ) = @_; + + return $self->get_counter51_value( $item, $type ); +} + +=head3 _get_COUNTER_row_usages + +Returns the total and monthly usages for a row + +=cut + +sub _get_COUNTER_row_usages { + my ( $self, $row, $metric_type, $access_type ) = @_; + + my @usage_months = $self->_get_usage_months( $self->{sushi}->{header} ); + + my @usage_months_fields = (); + my $count_total = 0; + + foreach my $usage_month (@usage_months) { + my $month_is_empty = 1; + + my $usage_to_add; + if ($access_type) { + my @access_type_performances = + grep { $_->{Access_Type} eq $access_type } @{ $row->{Attribute_Performance} }; + $usage_to_add = $access_type_performances[0]->{Performance}->{$metric_type}->{$usage_month}; + } else { + $usage_to_add = $row->{Attribute_Performance}->[0]->{Performance}->{$metric_type}->{$usage_month}; + } + if ( defined $usage_to_add ) { + $count_total += $usage_to_add; + $month_is_empty = 0; + push( @usage_months_fields, $usage_to_add ); + } + + if ($month_is_empty) { + push( @usage_months_fields, 0 ); + } + } + + return ( $count_total, @usage_months_fields ); +} + +=head3 get_SUSHI_metric_types + + Given a SUSHI report row, return a sorted list of all the metric types + +=cut + +sub get_SUSHI_metric_types { + my ( $self, $report_row ) = @_; + + my @metric_types = (); + @metric_types = keys %{ $report_row->{Attribute_Performance}->[0]->{Performance} }; + @metric_types = sort @metric_types; + return @metric_types; +} + +=head3 _COUNTER_report_body + +Return the COUNTER report body as an array + +=cut + +sub _COUNTER_report_body { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my $body = $self->{sushi}->{body}; + + my @report_body = (); + + foreach my $report_row ( @{$body} ) { + my @metric_types = $self->get_SUSHI_metric_types($report_row); + + my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); + my $use_access_type = grep ( /Access_Type/, @{$specific_fields} ) ? 1 : 0; + my $use_yop = grep ( /YOP/, @{$specific_fields} ) ? 1 : 0; + my $use_data_type = grep ( /Data_Type/, @{$specific_fields} ) ? 1 : 0; + + if ($use_data_type) { + my %data_types = map { $_->{Data_Type} => 1 } @{ $report_row->{Attribute_Performance} }; + my @data_types = sort keys %data_types; + + foreach my $data_type (@data_types) { + + if ($use_yop) { + my %yops = map { $_->{YOP} => 1 } @{ $report_row->{Attribute_Performance} }; + my @yops = sort { $b <=> $a } keys %yops; + foreach my $yop (@yops) { + + if ($use_access_type) { + my @access_types = map { $_->{"Access_Type"} } @{ $report_row->{Attribute_Performance} }; + foreach my $access_type (@access_types) { + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( + @report_body, + $self->_COUNTER_report_row( + $report_row, $metric_type, $access_type, $yop, + $data_type + ) + ); + } + } + } else { + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( + @report_body, + $self->_COUNTER_report_row( $report_row, $metric_type, undef, $yop, $data_type ) + ); + } + } + } + } else { + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( + @report_body, + $self->_COUNTER_report_row( $report_row, $metric_type, undef, undef, $data_type ) + ); + } + } + } + } elsif ($use_yop) { + my %yops = map { $_->{YOP} => 1 } @{ $report_row->{Attribute_Performance} }; + my @yops = sort { $b <=> $a } keys %yops; + foreach my $yop (@yops) { + + if ($use_access_type) { + my @access_types = map { $_->{"Access_Type"} } @{ $report_row->{Attribute_Performance} }; + foreach my $access_type (@access_types) { + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( + @report_body, + $self->_COUNTER_report_row( $report_row, $metric_type, $access_type, $yop ) + ); + } + } + } else { + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( @report_body, $self->_COUNTER_report_row( $report_row, $metric_type, undef, $yop ) ); + } + } + } + + } elsif ($use_access_type) { + my @access_types = map { $_->{"Access_Type"} } @{ $report_row->{Attribute_Performance} }; + foreach my $access_type (@access_types) { + + # Add one report row for each metric_type we're working with + foreach my $metric_type (@metric_types) { + push( @report_body, $self->_COUNTER_report_row( $report_row, $metric_type, $access_type ) ); + } + } + } else { + foreach my $metric_type (@metric_types) { + push( @report_body, $self->_COUNTER_report_row( $report_row, $metric_type ) ); + } + } + + } + + return @report_body; +} + +=head3 _COUNTER_title_report_row + +Return a COUNTER title for the COUNTER titles report body +https://cop5.countermetrics.org/en/5.1.0.1/04-reports/03-title-reports.html#column-headings-elements + +=cut + +sub _COUNTER_title_report_row { + my ( $self, $title_row, $metric_type, $total_usage, $monthly_usages, $access_type, $yop, $data_type ) = @_; + + my $header = $self->{sushi}->{header}; + my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); + my $access_type_to_use = $access_type ? $access_type : ( $title_row->{Access_Type} || "" ); + my $yop_to_use = $yop ? $yop : ( $title_row->{YOP} || "" ); + my $data_type_to_use = $data_type ? $data_type : ( $title_row->{Data_Type} || "" ); + + return ( + [ + # Title + $title_row->{Title} || "", + + # Publisher + $title_row->{Publisher} || "", + + # Publisher_ID + $self->_get_SUSHI_Type_Value( $title_row->{Publisher_ID}, "ISNI" ) || "", + + # Platform + $title_row->{Platform} || "", + + # DOI + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "DOI" ) || "", + + # Proprietary_ID + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Proprietary" ) || "", + + # ISBN + grep ( /ISBN/, @{$specific_fields} ) + ? ( $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "ISBN" ) || "" ) + : (), + + # Print_ISSN + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Print_ISSN" ) || "", + + # Online_ISSN + $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Online_ISSN" ) || "", + + # URI - FIXME: What goes in URI? + "", + + # Data_Type + grep ( /Data_Type/, @{$specific_fields} ) ? $data_type_to_use : (), + + # YOP + grep ( /YOP/, @{$specific_fields} ) ? $yop_to_use : (), + + # Access_Type + grep ( /Access_Type/, @{$specific_fields} ) ? $access_type_to_use : (), + + # Metric_Type + $metric_type, + + # Report_Period_Total + $total_usage, + + # Monthly usage entries + @{$monthly_usages} + ] + ); +} + +=head3 _COUNTER_titles_report_column_headings + +Return titles report column headings + +=cut + +sub _COUNTER_titles_report_column_headings { + my ($self) = @_; + + my $header = $self->{sushi}->{header}; + my @month_headings = $self->_get_usage_months( $header, 1 ); + my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); + + return ( + [ + "Title", + "Publisher", + "Publisher_ID", + "Platform", + "DOI", + "Proprietary_ID", + grep ( /ISBN/, @{$specific_fields} ) ? ("ISBN") : (), + "Print_ISSN", + "Online_ISSN", + "URI", + grep ( /Data_Type/, @{$specific_fields} ) ? ("Data_Type") : (), + + #"Section_Type", #TODO: Only if requested (?) + grep ( /YOP/, @{$specific_fields} ) ? ("YOP") : (), + grep ( /Access_Type/, @{$specific_fields} ) ? ("Access_Type") : (), + + #"Access_Method", #TODO: Only if requested (?) + "Metric_Type", + "Reporting_Period_Total", + + # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true + @month_headings + ] + ); +} + +=head3 get_report_type_specific_fields + +Returns the specific fields for a given report_type + +=cut + +sub get_report_type_specific_fields { + my ( $self, $report_type ) = @_; + + my %report_type_map = ( + "TR_B1" => [ 'YOP', 'Data_Type', 'ISBN' ], + "TR_B2" => [ 'YOP', 'Data_Type', 'ISBN' ], + "TR_B3" => [ 'YOP', 'Data_Type', 'Access_Type', 'ISBN' ], + "TR_J3" => ['Access_Type'], + "TR_J4" => ['YOP'], + "IR_A1" => [ + 'Authors', 'Publication_Date', 'Article_Version', 'Print_ISSN', 'Online_ISSN', 'Parent_Title', + 'Parent_Authors', 'Parent_Article_Version', 'Parent_DOI', 'Parent_Proprietary_ID', 'Parent_Print_ISSN', + 'Parent_Online_ISSN', 'Parent_URI', 'Access_Type' + ], + ); + + return $report_type_map{$report_type}; + +} + +1; diff --git a/Koha/ERM/EUsage/CounterFile.pm b/Koha/ERM/EUsage/CounterFile.pm index 2d5d80f15cd..18a653c2de6 100644 --- a/Koha/ERM/EUsage/CounterFile.pm +++ b/Koha/ERM/EUsage/CounterFile.pm @@ -85,6 +85,12 @@ Receive background_job_callbacks to be able to update job progress sub store { my ( $self, $background_job_callbacks ) = @_; + # DOCS: + # COUNTER 5 CSV examples: + # https://cop5.countermetrics.org/en/5.0.3/appendices/h-sample-counter-master-reports-and-standard-views.html + # COUNTER 5.1 CSV examples: + # https://cop5.projectcounter.org/en/5.1/appendices/g-sample-counter-reports-and-standard-views.html + $self->_set_report_type_from_file; my $result = $self->SUPER::store; @@ -227,7 +233,10 @@ sub _add_monthly_usage_entries { my $usage_data_provider = $self->get_usage_data_provider; my $usage_object_info = $self->_get_usage_object_id_hash($usage_object); - my $specific_fields = Koha::ERM::EUsage::SushiCounter->get_report_type_specific_fields( $self->type ); + my $specific_fields = Koha::ERM::EUsage::SushiCounter->get_report_type_specific_fields_by_release( + $self->type, + $self->get_COUNTER_report_release->{header_value} + ); $usage_object->monthly_usages( [ @@ -262,7 +271,10 @@ sub _add_yearly_usage_entries { my $usage_data_provider = $self->get_usage_data_provider; my $usage_object_info = $self->_get_usage_object_id_hash($usage_object); - my $specific_fields = Koha::ERM::EUsage::SushiCounter->get_report_type_specific_fields( $self->type ); + my $specific_fields = Koha::ERM::EUsage::SushiCounter->get_report_type_specific_fields_by_release( + $self->type, + $self->get_COUNTER_report_release->{header_value} + ); while ( my ( $year, $usage ) = each( %{$yearly_usages} ) ) { @@ -303,21 +315,12 @@ A I exception is thrown sub validate { my ($self) = @_; - open my $fh, "<", \$self->file_content or die; - my $csv = Text::CSV_XS->new( { binary => 1, always_quote => 1, eol => $/, decode_utf8 => 1 } ); - - $csv->column_names(qw( header_key header_value )); - my @header_rows = $csv->getline_hr_all( $fh, 0, 12 ); - my @header = $header_rows[0]; - - my @release_row = - map( $_->{header_key} eq 'Release' ? $_ : (), @{ $header[0] } ); - my $release = $release_row[0]; + my $release = $self->get_COUNTER_report_release; # TODO: Validate that there is an empty row between header and body Koha::Exceptions::ERM::EUsage::CounterFile::UnsupportedRelease->throw - if $release && $release->{header_value} != 5; + if $release && ( $release->{header_value} != 5 && $release->{header_value} ne '5.1' ); } @@ -344,6 +347,27 @@ sub _set_report_type_from_file { $self->type( $report->{header_value} ); } +=head3 get_COUNTER_report_release + +Retrieves the COUNTER report release for the given file_content + +=cut + +sub get_COUNTER_report_release { + my ($self) = @_; + + open my $fh, "<", \$self->file_content or die; + my $csv = Text::CSV_XS->new( { binary => 1, always_quote => 1, eol => $/, decode_utf8 => 1 } ); + + $csv->column_names(qw( header_key header_value )); + my @header_rows = $csv->getline_hr_all( $fh, 0, 12 ); + my @header = $header_rows[0]; + + my @release_row = + map( $_->{header_key} eq 'Release' ? $_ : (), @{ $header[0] } ); + return $release_row[0]; +} + =head3 _get_rows_from_COUNTER_file Returns array of rows from COUNTER file @@ -356,7 +380,12 @@ sub _get_rows_from_COUNTER_file { open my $fh, "<", \$self->file_content or die; my $csv = Text::CSV_XS->new( { binary => 1, always_quote => 1, eol => $/, decode_utf8 => 1 } ); - my $header_columns = $csv->getline_all( $fh, 13, 1 ); + my $release = $self->get_COUNTER_report_release; + $release = $release->{header_value}; + + my $header_rows = $release eq '5.1' ? 14 : 13; + + my $header_columns = $csv->getline_all( $fh, $header_rows, 1 ); $csv->column_names( @{$header_columns}[0] ); # Get all rows from 14th onward @@ -588,7 +617,10 @@ sub _add_usage_object_entry { my ( $self, $row ) = @_; my $usage_data_provider = $self->get_usage_data_provider; - my $specific_fields = Koha::ERM::EUsage::SushiCounter->get_report_type_specific_fields( $self->type ); + my $specific_fields = Koha::ERM::EUsage::SushiCounter->get_report_type_specific_fields_by_release( + $self->type, + $self->get_COUNTER_report_release->{header_value} + ); if ( $self->type =~ /PR/i ) { my $new_usage_platform = Koha::ERM::EUsage::UsagePlatform->new( diff --git a/Koha/ERM/EUsage/SushiCounter.pm b/Koha/ERM/EUsage/SushiCounter.pm index 54763206430..349245d5f23 100644 --- a/Koha/ERM/EUsage/SushiCounter.pm +++ b/Koha/ERM/EUsage/SushiCounter.pm @@ -41,15 +41,26 @@ Koha::ERM::EUsage::SushiCounter - Koha SushiCounter Object class sub new { my ( $class, $params ) = @_; - my $self = $class->SUPER::new; - $self->{sushi} = { + my $counter_release; + if ( $params->{response}->{Report_Header}->{Release} eq "5" ) { + require Koha::ERM::EUsage::COUNTER::5; + $counter_release = Koha::ERM::EUsage::COUNTER::5->new; + } elsif ( $params->{response}->{Report_Header}->{Release} eq "5.1" ) { + require Koha::ERM::EUsage::COUNTER::5_1; + $counter_release = Koha::ERM::EUsage::COUNTER::5_1->new; + } else { + + #TODO: Throw an exception stating release not found / not supported! + } + + $counter_release->{sushi} = { header => $params->{response}->{Report_Header}, body => $params->{response}->{Report_Items} }; #TODO: Handle empty $self->{sushi}->{body} here! - return $self; + return $counter_release; } =head3 get_COUNTER_from_SUSHI @@ -92,47 +103,6 @@ sub _build_COUNTER_report_file { } -=head3 _COUNTER_report_header - -Return a COUNTER report header -https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html - -=cut - -sub _COUNTER_report_header { - my ($self) = @_; - - my $header = $self->{sushi}->{header}; - - my @metric_types_string = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Metric_Type" ); - - my $begin_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "Begin_Date" ); - my $end_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, "End_Date" ); - - return ( - [ Report_Name => $header->{Report_Name} || "" ], - [ Report_ID => $header->{Report_ID} || "" ], - [ Release => $header->{Release} || "" ], - [ Institution_Name => $header->{Institution_Name} || "" ], - [ Institution_ID => join( "; ", map( $_->{Type} . ":" . $_->{Value}, @{ $header->{Institution_ID} } ) ) || "" ], - [ Metric_Types => join( "; ", split( /\|/, $metric_types_string[0] ) ) || "" ], - [ Report_Filters => join( "; ", map( $_->{Name} . ":" . $_->{Value}, @{ $header->{Report_Filters} } ) ) || "" ], - - #TODO: Report_Attributes may need parsing, test this with a SUSHI response that provides it - [ Report_Attributes => $header->{Report_Attributes} || "" ], - [ - Exceptions => join( - "; ", map( $_->{Code} . ": " . $_->{Message} . " (" . $_->{Data} . ")", @{ $header->{Exceptions} } ) - ) - || "" - ], - [ Reporting_Period => "Begin_Date=" . $begin_date . "; End_Date=" . $end_date ], - [ Created => $header->{Created} || "" ], - [ Created_By => $header->{Created_By} || "" ], - [""] #empty 13th line (COUNTER 5) - ); -} - =head3 _COUNTER_item_report_row Return a COUNTER item for the COUNTER items report body @@ -274,71 +244,6 @@ sub _COUNTER_platform_report_row { ); } -=head3 _COUNTER_title_report_row - -Return a COUNTER title for the COUNTER titles report body -https://cop5.projectcounter.org/en/5.0.2/04-reports/03-title-reports.html#column-headings-elements - -=cut - -sub _COUNTER_title_report_row { - my ( $self, $title_row, $metric_type, $total_usage, $monthly_usages ) = @_; - - my $header = $self->{sushi}->{header}; - my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); - - return ( - [ - # Title - $title_row->{Title} || "", - - # Publisher - $title_row->{Publisher} || "", - - # Publisher_ID - $self->_get_SUSHI_Type_Value( $title_row->{Publisher_ID}, "ISNI" ) || "", - - # Platform - $title_row->{Platform} || "", - - # DOI - $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "DOI" ) || "", - - # Proprietary_ID - $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Proprietary" ) || "", - - # ISBN - grep ( /ISBN/, @{$specific_fields} ) - ? ( $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "ISBN" ) || "" ) - : (), - - # Print_ISSN - $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Print_ISSN" ) || "", - - # Online_ISSN - $self->_get_SUSHI_Type_Value( $title_row->{Item_ID}, "Online_ISSN" ) || "", - - # URI - FIXME: What goes in URI? - "", - - # YOP - grep ( /YOP/, @{$specific_fields} ) ? ( $title_row->{YOP} || "" ) : (), - - # Access_Type - grep ( /Access_Type/, @{$specific_fields} ) ? ( $title_row->{Access_Type} || "" ) : (), - - # Metric_Type - $metric_type, - - # Report_Period_Total - $total_usage, - - # Monthly usage entries - @{$monthly_usages} - ] - ); -} - =head3 _COUNTER_report_row Return a COUNTER row for the COUNTER report body @@ -346,11 +251,11 @@ Return a COUNTER row for the COUNTER report body =cut sub _COUNTER_report_row { - my ( $self, $report_row, $metric_type ) = @_; + my ( $self, $report_row, $metric_type, $access_type, $yop, $data_type ) = @_; my $header = $self->{sushi}->{header}; - my ( $total_usage, @monthly_usages ) = $self->_get_COUNTER_row_usages( $report_row, $metric_type ); + my ( $total_usage, @monthly_usages ) = $self->_get_COUNTER_row_usages( $report_row, $metric_type, $access_type ); if ( $header->{Report_ID} =~ /PR/i ) { return $self->_COUNTER_platform_report_row( @@ -360,115 +265,15 @@ sub _COUNTER_report_row { } elsif ( $header->{Report_ID} =~ /DR/i ) { return $self->_COUNTER_database_report_row( $report_row, $metric_type, $total_usage, \@monthly_usages ); } elsif ( $header->{Report_ID} =~ /IR/i ) { + + #TODO: Send $access_type to _COUNTER_item_report_row like we're doing with _COUNTER_title_report_row return $self->_COUNTER_item_report_row( $report_row, $metric_type, $total_usage, \@monthly_usages ); } elsif ( $header->{Report_ID} =~ /TR/i ) { - return $self->_COUNTER_title_report_row( $report_row, $metric_type, $total_usage, \@monthly_usages ); - } -} - -=head3 _get_COUNTER_row_usages - -Returns the total and monthly usages for a row - -=cut - -sub _get_COUNTER_row_usages { - my ( $self, $row, $metric_type ) = @_; - - my @usage_months = $self->_get_usage_months( $self->{sushi}->{header} ); - - my @usage_months_fields = (); - my $count_total = 0; - - foreach my $usage_month (@usage_months) { - my $month_is_empty = 1; - - foreach my $performance ( @{ $row->{Performance} } ) { - my $period = $performance->{Period}; - my $period_usage_month = substr( $period->{Begin_Date}, 0, 7 ); - - my $instances = $performance->{Instance}; - my @metric_type_count = - map( $_->{Metric_Type} eq $metric_type ? $_->{Count} : (), - @{$instances} ); - - if ( $period_usage_month eq $usage_month && $metric_type_count[0] ) { - push( @usage_months_fields, $metric_type_count[0] ); - $count_total += $metric_type_count[0]; - $month_is_empty = 0; - } - } - - if ($month_is_empty) { - push( @usage_months_fields, 0 ); - } - } - return ( $count_total, @usage_months_fields ); -} - -=head3 _COUNTER_report_body - -Return the COUNTER report body as an array - -=cut - -sub _COUNTER_report_body { - my ($self) = @_; - - my $header = $self->{sushi}->{header}; - my $body = $self->{sushi}->{body}; - - my @report_body = (); - - foreach my $report_row ( @{$body} ) { - - my @metric_types = (); - - # Grab all metric_types this SUSHI result has statistics for - foreach my $performance ( @{ $report_row->{Performance} } ) { - my @SUSHI_metric_types = - map( $_->{Metric_Type}, @{ $performance->{Instance} } ); - - foreach my $sushi_metric_type (@SUSHI_metric_types) { - push( @metric_types, $sushi_metric_type ) unless grep { $_ eq $sushi_metric_type } @metric_types; - } - } - - # Add one report row for each metric_type we're working with - foreach my $metric_type (@metric_types) { - push( @report_body, $self->_COUNTER_report_row( $report_row, $metric_type ) ); - } + return $self->_COUNTER_title_report_row( + $report_row, $metric_type, $total_usage, \@monthly_usages, + $access_type, $yop, $data_type + ); } - - return @report_body; -} - -=head3 _get_SUSHI_Name_Value - -Returns "Value" of a given "Name" - -=cut - -sub _get_SUSHI_Name_Value { - my ( $self, $item, $name ) = @_; - - my @value = map( $_->{Name} eq $name ? $_->{Value} : (), @{$item} ); - - return $value[0]; -} - -=head3 _get_SUSHI_Type_Value - -Returns "Value" of a given "Type" - -=cut - -sub _get_SUSHI_Type_Value { - my ( $self, $item, $type ) = @_; - - my @value = map( $_->{Type} eq $type ? $_->{Value} : (), @{$item} ); - - return $value[0]; } =head3 _COUNTER_report_column_headings @@ -617,45 +422,22 @@ sub _COUNTER_platforms_report_column_headings { ); } -=head3 _COUNTER_titles_report_column_headings +=head3 get_report_type_specific_fields_by_release -Return titles report column headings +Returns the specific fields for a given report_type =cut -sub _COUNTER_titles_report_column_headings { - my ($self) = @_; - - my $header = $self->{sushi}->{header}; - my @month_headings = $self->_get_usage_months( $header, 1 ); - my $specific_fields = $self->get_report_type_specific_fields( $header->{Report_ID} ); - - return ( - [ - "Title", - "Publisher", - "Publisher_ID", - "Platform", - "DOI", - "Proprietary_ID", - grep ( /ISBN/, @{$specific_fields} ) ? ("ISBN") : (), - "Print_ISSN", - "Online_ISSN", - "URI", - - #"Data_Type", #TODO: Only if requested (?) - #"Section_Type", #TODO: Only if requested (?) - grep ( /YOP/, @{$specific_fields} ) ? ("YOP") : (), - grep ( /Access_Type/, @{$specific_fields} ) ? ("Access_Type") : (), +sub get_report_type_specific_fields_by_release { + my ( $self, $report_type, $counter_release ) = @_; - #"Access_Method", #TODO: Only if requested (?) - "Metric_Type", - "Reporting_Period_Total", - - # @month_headings in "Mmm-yyyy" format. TODO: Show unless Exclude_Monthly_Details=true - @month_headings - ] - ); + if ( $counter_release eq "5" ) { + require Koha::ERM::EUsage::COUNTER::5; + return Koha::ERM::EUsage::COUNTER::5->get_report_type_specific_fields($report_type); + } elsif ( $counter_release eq "5.1" ) { + require Koha::ERM::EUsage::COUNTER::5_1; + return Koha::ERM::EUsage::COUNTER::5_1->get_report_type_specific_fields($report_type); + } } =head3 _get_usage_months @@ -672,11 +454,11 @@ sub _get_usage_months { "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" ); - my @begin_date = map( $_->{Name} eq "Begin_Date" ? $_->{Value} : (), @{ $header->{Report_Filters} } ); + my @begin_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, 'Begin_Date' ); my $begin_month = substr( $begin_date[0], 5, 2 ); my $begin_year = substr( $begin_date[0], 0, 4 ); - my @end_date = map( $_->{Name} eq "End_Date" ? $_->{Value} : (), @{ $header->{Report_Filters} } ); + my @end_date = $self->_get_SUSHI_Name_Value( $header->{Report_Filters}, 'End_Date' ); my $end_month = substr( $end_date[0], 5, 2 ); my $end_year = substr( $end_date[0], 0, 4 ); @@ -699,32 +481,6 @@ sub _get_usage_months { return @month_headings; } -=head3 get_report_type_specific_fields - -Returns the specific fields for a given report_type - -=cut - -sub get_report_type_specific_fields { - my ( $self, $report_type ) = @_; - - my %report_type_map = ( - "TR_B1" => [ 'YOP', 'ISBN' ], - "TR_B2" => [ 'YOP', 'ISBN' ], - "TR_B3" => [ 'YOP', 'Access_Type', 'ISBN' ], - "TR_J3" => ['Access_Type'], - "TR_J4" => ['YOP'], - "IR_A1" => [ - 'Authors', 'Publication_Date', 'Article_Version', 'Print_ISSN', 'Online_ISSN', 'Parent_Title', - 'Parent_Authors', 'Parent_Article_Version', 'Parent_DOI', 'Parent_Proprietary_ID', 'Parent_Print_ISSN', - 'Parent_Online_ISSN', 'Parent_URI', 'Access_Type' - ], - ); - - return $report_type_map{$report_type}; - -} - =head3 _type =cut -- 2.39.5