From d71ee741a714b38d4b710a8a275ce653ecda6262 Mon Sep 17 00:00:00 2001 From: Cori Lynn Arnold Date: Mon, 15 Oct 2018 18:10:40 +0000 Subject: [PATCH] Bug 18823: Improvements to import batch searching and enabled/disabled Rebase. Signed-off-by: cori Rebased-on: 2019-03-20 Alex Arnaud --- Koha/MetaSearcher.pm | 168 ++++++++++++++++----- .../lib/koha/cateditor/preferences.js | 1 + .../intranet-tmpl/lib/koha/cateditor/search.js | 32 ++-- .../prog/en/includes/cateditor-ui.inc | 94 +++++++----- .../prog/en/modules/cataloguing/editor.tt | 29 ++-- svc/cataloguing/metasearch | 17 ++- 6 files changed, 233 insertions(+), 108 deletions(-) diff --git a/Koha/MetaSearcher.pm b/Koha/MetaSearcher.pm index 65f7239..366e98b 100644 --- a/Koha/MetaSearcher.pm +++ b/Koha/MetaSearcher.pm @@ -56,9 +56,9 @@ sub new { } sub handle_hit { - my ( $self, $index, $server, $marcrecord ) = @_; + my ( $self, $index, $server, $hit ) = @_; - my $record = Koha::MetadataRecord->new( { schema => 'marc', record => $marcrecord } ); + my $record = Koha::MetadataRecord->new( { schema => 'marc', record => $hit->{record} } ); my %fetch = ( title => 'biblio.title', @@ -79,15 +79,15 @@ sub handle_hit { $metadata->{date} //= $metadata->{date2}; push @{ $self->{results} }, { + %$hit, server => $server, index => $index, - record => $marcrecord, metadata => $metadata, }; } sub search { - my ( $self, $server_ids, $query ) = @_; + my ( $self, $server_ids, $terms ) = @_; my $resultset_expiry = 300; @@ -142,7 +142,7 @@ sub search { } } - $select->add( $self->_start_worker( $server, $query ) ); + $select->add( $self->_start_worker( $server, $terms ) ); } # Handle these while the servers are searching @@ -182,8 +182,102 @@ sub search { return $stats; } +our $_pqf_mapping = { + author => '1=1004', + 'cn-dewey' => '1=13', + 'cn-lc' => '1=16', + date => '1=30', + isbn => '1=7', + issn => '1=8', + keyword => '1=1016', + lccn => '1=9', + 'local-number' => '1=12', + 'music-identifier' => '1=51', + 'standard-identifier' => '1=1007', + subject => '1=21', + title => '1=4', +}; + +our $_batch_db_mapping = { + author => 'import_biblios.author', + isbn => 'import_biblios.isbn', + issn => 'import_biblios.issn', + 'local-number' => 'import_biblios.control_number', + title => 'import_biblios.title', +}; + +our $_batch_db_text_columns = { + author => 1, + keyword => 1, + title => 1, +}; + +sub _pqf_query_from_terms { + my ( $terms ) = @_; + + my $query; + + while ( my ( $index, $value ) = each %$terms ) { + $value =~ s/"/\\"/; + + my $term = '@attr ' . $_pqf_mapping->{$index} . ' "' . $value . '"'; + + if ( $query ) { + $query = '@and ' . $query . ' ' . $term; + } else { + $query = $term; + } + } + + return $query; +} + +sub _db_query_get_match_conditions { + my ( $index, $value ) = @_; + + if ( $value =~ /\*/ ) { + $value =~ s/\*/%/; + return { -like => $value }; + } elsif ( $_batch_db_text_columns->{$index} ) { + return map +{ -regexp => '[[:<:]]' . $_ . '[[:>:]]' }, split( /\s+/, $value ); + } else { + return $value; + } +} + +sub _batch_db_query_from_terms { + my ( $import_batch_id, $terms ) = @_; + + my @db_terms; + + while ( my ( $index, $value ) = each %$terms ) { + if ( $index eq 'keyword' ) { + my @term; + + while ( my ( $index, $db_column ) = each %$_batch_db_mapping ) { + push @term, { $db_column => [ _db_query_get_match_conditions( $index, $value ) ] }; + } + + # These are implicitly joined with OR because the arrayref doesn't start with -and + push @db_terms, \@term; + } elsif ( $_batch_db_mapping->{$index} ) { + push @db_terms, $_batch_db_mapping->{$index} => [ -and => _db_query_get_match_conditions( $index, $value ) ]; + } else { + # No such index, we should fail + return undef; + } + } + + return { + -and => [ + import_batch_id => $import_batch_id, + @db_terms + ], + }, +} + sub _start_worker { - my ( $self, $server, $query ) = @_; + my ( $self, $server, $terms ) = @_; pipe my $readfh, my $writefh; # Accessing the cache or Koha database after the fork is risky, so get any resources we need @@ -207,6 +301,8 @@ sub _start_worker { my $connection; my ( $num_hits, $num_fetched, $hits, $results ); + my $pqf_query = _pqf_query_from_terms( $terms ); + eval { if ( $server->{type} eq 'z3950' ) { my $zoptions = ZOOM::Options->new(); @@ -220,66 +316,64 @@ sub _start_worker { $connection = ZOOM::Connection->create($zoptions); $connection->connect( $server->{host}, $server->{port} ); - $results = $connection->search_pqf( $query ); # Starts the search + $results = $connection->search_pqf( $pqf_query ); # Starts the search } elsif ( $server->{type} eq 'koha' ) { $connection = C4::Context->Zconn( $server->{extra} ); - $results = $connection->search_pqf( $query ); # Starts the search + $results = $connection->search_pqf( $pqf_query ); # Starts the search } elsif ( $server->{type} eq 'batch' ) { $server->{encoding} = 'utf-8'; } }; if ($@) { store_fd { - error => $connection ? $connection->exception() : $@, + error => $connection ? $connection->exception()->message() : $@, server => $server, }, $writefh; exit; } + my $error; if ( $server->{type} eq 'batch' ) { - # TODO: actually handle PQF - $query =~ s/@\w+ (?:\d+=\d+ )?//g; - $query =~ s/"//g; - my $schema = Koha::Database->new->schema; - $schema->storage->debug(1); - my $match_condition = [ map +{ -like => '%' . $_ . '%' }, split( /\s+/, $query ) ]; - $hits = [ $schema->resultset('ImportRecord')->search( - { - import_batch_id => $server->{extra}, - -or => [ - { 'import_biblios.title' => $match_condition }, - { 'import_biblios.author' => $match_condition }, - { 'import_biblios.isbn' => $match_condition }, - { 'import_biblios.issn' => $match_condition }, - ], - }, - { - join => [ qw( import_biblios ) ], - rows => $self->{fetch}, - } - )->get_column( 'marc' )->all ]; - - $num_hits = $num_fetched = scalar @$hits; + my $db_query = _batch_db_query_from_terms( $server->{extra}, $terms ); + + if ( $db_query ) { + $hits = [ $schema->resultset('ImportRecord')->search( + $db_query, + { + columns => [ 'import_record_id', { record => 'marc' } ], + join => [ 'import_biblios' ], + offset => $self->{offset}, + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + rows => $self->{fetch}, + } + )->all ]; + + $num_hits = $num_fetched = scalar @$hits; + } else { + $error = "Invalid search"; + } } else { $num_hits = $results->size; $num_fetched = ( $self->{offset} + $self->{fetch} ) < $num_hits ? $self->{fetch} : $num_hits; - $hits = [ map { $_->raw() } @{ $results->records( $self->{offset}, $num_fetched, 1 ) } ]; + $hits = [ map +{ record => $_->raw() }, @{ $results->records( $self->{offset}, $num_fetched, 1 ) } ]; + + $error = $connection->exception()->message() if ( !@$hits && $connection && $connection->exception() ); } - if ( !@$hits && $connection && $connection->exception() ) { + if ( $error ) { store_fd { - error => $connection->exception(), + error => $error, server => $server, }, $writefh; exit; } if ( $server->{type} eq 'koha' ) { - $hits = [ map { C4::Search::new_record_from_zebra( $server->{extra}, $_ ) } @$hits ]; + $hits = [ map +{ %$_, record => C4::Search::new_record_from_zebra( $server->{extra}, $_->{record} ) }, @$hits ]; } else { - $hits = [ map { $self->_import_record( $_, $marcflavour, $server->{encoding} ? $server->{encoding} : "iso-5426" ) } @$hits ]; + $hits = [ map +{ %$_, record => $self->_import_record( $_->{record}, $marcflavour, $server->{encoding} ? $server->{encoding} : "iso-5426" ) }, @$hits ]; } store_fd { diff --git a/koha-tmpl/intranet-tmpl/lib/koha/cateditor/preferences.js b/koha-tmpl/intranet-tmpl/lib/koha/cateditor/preferences.js index d684934..00121d8 100644 --- a/koha-tmpl/intranet-tmpl/lib/koha/cateditor/preferences.js +++ b/koha-tmpl/intranet-tmpl/lib/koha/cateditor/preferences.js @@ -30,6 +30,7 @@ define( function() { Preferences.user = $.extend( { // Preference defaults enabledBatches: {}, + enabledSearchBatches: {}, fieldWidgets: true, font: 'monospace', fontSize: '1em', diff --git a/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js b/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js index f73d029..532fa96 100644 --- a/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js +++ b/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js @@ -42,22 +42,7 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { Init: function( options ) { _options = options; }, - JoinTerms: function( terms ) { - var q = ''; - - $.each( terms, function( i, term ) { - var term = '@attr ' + _pqfMapping[ term[0] ] + ' "' + term[1].replace( '"', '\\"' ) + '"' - - if ( q ) { - q = '@and ' + q + ' ' + term; - } else { - q = term; - } - } ); - - return q; - }, - Run: function( servers, q, options ) { + Run: function( servers, terms, options ) { options = $.extend( { offset: 0, page_size: 20, @@ -67,10 +52,16 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { _records = {}; _last = { servers: servers, - q: q, + terms: terms, options: options, }; + var newTerms = {}; + $.each( terms, function( index, value ) { + newTerms[ "term-" + index ] = value; + } ); + terms = newTerms; + var itemTag = KohaBackend.GetSubfieldForKohaField('items.itemnumber')[0]; $.each( servers, function ( id, info ) { @@ -81,15 +72,14 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { $.get( '/cgi-bin/koha/svc/cataloguing/metasearch', - { - q: q, + $.extend( { servers: Search.includedServers.join( ',' ), offset: options.offset, page_size: options.page_size, sort_direction: options.sort_direction, sort_key: options.sort_key, resultset: options.resultset, - } + }, terms ) ) .done( function( data ) { _last.options.resultset = data.resultset; @@ -115,7 +105,7 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { Fetch: function( options ) { if ( !_last ) return; $.extend( _last.options, options ); - return Search.Run( _last.servers, _last.q, _last.options ); + return Search.Run( _last.servers, _last.terms, _last.options ); } }; diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc index 91306ea..bb890a1 100644 --- a/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc +++ b/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc @@ -37,6 +37,7 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr 'koha:biblioserver': { name: _("Local catalog"), recordtype: 'biblio', + enabled: true, checked: false, }, [%- FOREACH batch = editable_batches -%] @@ -51,6 +52,7 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr name: '[% server.servername | html %]', recordtype: '[% server.recordtype | html %]', checked: [% server.checked ? 'true' : 'false' | html %], + enabled: true, }, [%- END -%] }; @@ -165,17 +167,17 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr $('#quicksearch .search-box').each( function() { shortcut.add( 'enter', $.proxy( function() { - var terms = []; + var terms = {}; $('#quicksearch .search-box').each( function() { if ( !this.value ) return; - terms.push( [ $(this).data('qualifier'), this.value ] ); + terms[ $(this).data('qualifier') ] = this.value; } ); - if ( !terms.length ) return; + if ( $.isEmptyObject(terms) ) return; - if ( Search.Run( z3950Servers, Search.JoinTerms(terms) ) ) { + if ( Search.Run( z3950Servers, terms ) ) { $("#search-overlay").show(); showResultsBox(); } @@ -366,6 +368,22 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr $( '#save-targets input[data-target-id="' + target_id + '"]' ).closest('li').toggle(enabled); } + function setSearchTargetChecked( server_id, checked ) { + if ( z3950Servers[server_id] == null ) return; + + z3950Servers[server_id].checked = checked; + $( '.search-target-list li[data-server-id="' + server_id + '"] input' ).prop('checked', checked); + } + + function setSearchTargetEnabled( server_id, enabled ) { + if ( !enabled ) { + setSearchTargetChecked( server_id, false ); + } + + z3950Servers[server_id].enabled = enabled; + $( '.search-target-list li[data-server-id="' + server_id + '"]' ).toggle(enabled); + } + function setSource(parts) { state.backend = parts[0]; state.recordID = parts[1]; @@ -466,23 +484,24 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr function showAdvancedSearch() { $('#advanced-search-servers').empty(); $.each( z3950Servers, function( server_id, server ) { + if ( !server.enabled ) return; $('#advanced-search-servers').append( '
  • ' ); } ); $('#advanced-search-ui').modal('show'); } function startAdvancedSearch() { - var terms = []; + var terms = {}; $('#advanced-search-ui .search-box').each( function() { if ( !this.value ) return; - terms.push( [ $(this).data('qualifier'), this.value ] ); + terms[ $(this).data('qualifier') ] = this.value; } ); - if ( !terms.length ) return; + if ( $.isEmptyObject(terms) ) return; - if ( Search.Run( z3950Servers, Search.JoinTerms(terms) ) ) { + if ( Search.Run( z3950Servers, terms ) ) { $('#advanced-search-ui').modal('hide'); $("#search-overlay").show(); showResultsBox(); @@ -555,19 +574,15 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr $.each( data.hits, function( undef, hit ) { backends.search.records[ hit.server + ':' + hit.index ] = hit.record; - switch ( hit.server ) { - case 'koha:biblioserver': - var bibnumField = hit.record.field( bibnumMap[0] ); - - if ( bibnumField && bibnumField.hasSubfield( bibnumMap[1] ) ) { - hit.id = 'catalog/' + bibnumField.subfield( bibnumMap[1] ); - break; - } - - // Otherwise, fallthrough + hit.id = 'search/' + hit.server + ':' + hit.index; + if ( hit.server == 'koha:biblioserver' ) { + var bibnumField = hit.record.field( bibnumMap[0] ); - default: - hit.id = 'search/' + hit.server + ':' + hit.index; + if ( bibnumField && bibnumField.hasSubfield( bibnumMap[1] ) ) { + hit.id = 'catalog/' + bibnumField.subfield( bibnumMap[1] ); + } + } else if ( /^batch:/.test(hit.server) && hit.import_record_id ) { + hit.id = hit.server + '/' + hit.import_record_id; } var result = ''; @@ -688,9 +703,14 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr switch (pref) { case 'enabledBatches': $.each( editable_batches, function( batch_id, batch ) { - $( '#batches-list li[data-batch-id=' + batch_id + '] input' )[0].checked = Preferences.user.enabledBatches[batch_id]; + $( '#batches-table tr[data-batch-id=' + batch_id + '] input.is-save-target' )[0].checked = Preferences.user.enabledBatches[batch_id]; setSaveTargetEnabled( 'batch:' + batch_id + '/', Preferences.user.enabledBatches[batch_id] || false ); } ); + case 'enabledSearchBatches': + $.each( editable_batches, function( batch_id, batch ) { + $( '#batches-table tr[data-batch-id=' + batch_id + '] input.is-searchable' )[0].checked = Preferences.user.enabledSearchBatches[batch_id]; + setSearchTargetEnabled( 'batch:' + batch_id, Preferences.user.enabledSearchBatches[batch_id] || false ); + } ); case 'fieldWidgets': $( '#set-field-widgets' ).text( value ? _("Show fields verbatim") : _("Show helpers for fixed and coded fields") ); break; @@ -719,13 +739,6 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr if ( saved_val != null ) setSaveTargetChecked( target_id, saved_val ); } ); break; - case 'selected_search_targets': - $.each( z3950Servers, function( server_id, server ) { - var saved_val = Preferences.user.selected_search_targets[server_id]; - - if ( saved_val != null ) server.checked = saved_val; - } ); - break; } } @@ -750,8 +763,13 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr switch (pref) { case 'enabledBatches': - _addLiveHandler( '#batches-list input', 'change', function() { - Preferences.user.enabledBatches[ $( this ).closest('li').data('batch-id') ] = this.checked; + _addLiveHandler( '#batches-table input.is-save-target', 'change', function() { + Preferences.user.enabledBatches[ $( this ).closest('tr').data('batch-id') ] = this.checked; + } ); + break; + case 'enabledSearchBatches': + _addLiveHandler( '#batches-table input.is-searchable', 'change', function() { + Preferences.user.enabledSearchBatches[ $( this ).closest('tr').data('batch-id') ] = this.checked; } ); break; case 'fieldWidgets': @@ -908,14 +926,16 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr }; // Build batch UI - var $batch_entry = $( '
  • ' + batch.name + '
  • ' ); + var $batch_entry = $( '' + batch.name + '' ); - var $batch_buttons = $('').appendTo($batch_entry); + var $batch_buttons = $('').appendTo($batch_entry); var $export_button = $( '' ).appendTo($batch_buttons).click( function() { - $('#batches-list .batch-export').hide(); + $('#batches-table .batch-export').hide(); $export_screen.show(); } ); + timestamp_pattern = '\\d{4}(0\\d|1[012])([012]\\d|3[01])(([01]\\d|2[0-3])[0-5]\\d[0-5]\\d)?' + var $export_screen = $( ' -
      + + + + + + + + + + + +
      BatchSearchable?Save target? 
      diff --git a/svc/cataloguing/metasearch b/svc/cataloguing/metasearch index 110732f..4e946d0 100755 --- a/svc/cataloguing/metasearch +++ b/svc/cataloguing/metasearch @@ -25,7 +25,7 @@ use Koha::MetaSearcher; my ( $query, $response ) = C4::Service->init( catalogue => 1 ); -my ( $query_string, $servers ) = C4::Service->require_params( 'q', 'servers' ); +my ( $servers ) = C4::Service->require_params( 'servers' ); my $server_errors = {}; @@ -35,19 +35,27 @@ my $offset = $query->param( 'offset' ) || 0; my $page_size = $query->param( 'page_size' ) || 20; my $fetched = $query->param( 'fetched' ) || 100; +my $terms = {}; + +foreach my $term ( $query->param ) { + next unless ( $term =~ /^term-([A-Za-z0-9-]+)/ ); + + $terms->{$1} = $query->param($term); +} + my $searcher = Koha::MetaSearcher->new( { fetched => $fetched, on_error => sub { my ( $server, $exception ) = @_; - $server_errors->{ $server->{id} } = $exception->message; + $server_errors->{ $server->{id} } = $exception; }, } ); $searcher->resultset( $query->param('resultset') ) if ( $query->param('resultset') ); my @server_ids = split( /,/, $servers ); -my $stats = $searcher->search( \@server_ids, $query_string ); +my $stats = $searcher->search( \@server_ids, $terms ); $searcher->sort( $sort_key, $sort_direction eq 'desc' ? -1 : 1 ); @@ -55,10 +63,11 @@ my @hits; foreach my $hit ( $searcher->results( $offset, $page_size ) ) { push @hits, { + %$hit, server => $hit->{server}->{id}, index => $hit->{index}, record => $hit->{record}->as_xml_record(), - metadata => $hit->{metadata} + metadata => $hit->{metadata}, }; } -- 2.7.4