Bugzilla – Attachment 66883 Details for
Bug 18823
Advanced editor - Rancor - add ability to edit records in import batches
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Help
|
New Account
|
Log In
[x]
|
Forgot Password
Login:
[x]
[patch]
Bug 18823: Rancor - Improvements to import batch searching
Bug-18823-Rancor---Improvements-to-import-batch-se.patch (text/plain), 20.37 KB, created by
Nick Clemens (kidclamp)
on 2017-09-06 14:38:37 UTC
(
hide
)
Description:
Bug 18823: Rancor - Improvements to import batch searching
Filename:
MIME Type:
Creator:
Nick Clemens (kidclamp)
Created:
2017-09-06 14:38:37 UTC
Size:
20.37 KB
patch
obsolete
>From 78ac659087d095d120de3e5931720278d6d4cd30 Mon Sep 17 00:00:00 2001 >From: Jesse Weaver <jweaver@bywatersolutions.com> >Date: Fri, 8 Jan 2016 14:46:44 -0700 >Subject: [PATCH] Bug 18823: Rancor - Improvements to import batch searching > >Refactoring and improving feedback >--- > Koha/MetaSearcher.pm | 168 ++++++++++++++++----- > .../intranet-tmpl/lib/koha/cateditor/search.js | 48 ++---- > .../prog/en/includes/cateditor-ui.inc | 43 ++---- > .../prog/en/modules/cataloguing/editor.tt | 12 +- > svc/cataloguing/metasearch | 17 ++- > 5 files changed, 177 insertions(+), 111 deletions(-) > >diff --git a/Koha/MetaSearcher.pm b/Koha/MetaSearcher.pm >index ccf50bd..3cc4977 100644 >--- a/Koha/MetaSearcher.pm >+++ b/Koha/MetaSearcher.pm >@@ -56,9 +56,9 @@ sub new { > } > > sub handle_hit { >- my ( $self, $index, $server, $marcrecord ) = @_; >+ my ( $self, $index, $server, $hit ) = @_; > >- my $record = Koha::MetadataRecord->new( { schema => 'marc', record => $marcrecord } ); >+ my $record = Koha::MetadataRecord->new( { schema => 'marc', record => $hit->{record} } ); > > my %fetch = ( > title => 'biblio.title', >@@ -79,15 +79,15 @@ sub handle_hit { > $metadata->{date} //= $metadata->{date2}; > > push @{ $self->{results} }, { >+ %$hit, > server => $server, > index => $index, >- record => $marcrecord, > metadata => $metadata, > }; > } > > sub search { >- my ( $self, $server_ids, $query ) = @_; >+ my ( $self, $server_ids, $terms ) = @_; > > my $resultset_expiry = 300; > >@@ -142,7 +142,7 @@ sub search { > } > } > >- $select->add( $self->_start_worker( $server, $query ) ); >+ $select->add( $self->_start_worker( $server, $terms ) ); > } > > # Handle these while the servers are searching >@@ -182,8 +182,102 @@ sub search { > return $stats; > } > >+our $_pqf_mapping = { >+ author => '1=1004', >+ 'cn-dewey' => '1=13', >+ 'cn-lc' => '1=16', >+ date => '1=30', >+ isbn => '1=7', >+ issn => '1=8', >+ keyword => '1=1016', >+ lccn => '1=9', >+ 'local-number' => '1=12', >+ 'music-identifier' => '1=51', >+ 'standard-identifier' => '1=1007', >+ subject => '1=21', >+ title => '1=4', >+}; >+ >+our $_batch_db_mapping = { >+ author => 'import_biblios.author', >+ isbn => 'import_biblios.isbn', >+ issn => 'import_biblios.issn', >+ 'local-number' => 'import_biblios.control_number', >+ title => 'import_biblios.title', >+}; >+ >+our $_batch_db_text_columns = { >+ author => 1, >+ keyword => 1, >+ title => 1, >+}; >+ >+sub _pqf_query_from_terms { >+ my ( $terms ) = @_; >+ >+ my $query; >+ >+ while ( my ( $index, $value ) = each %$terms ) { >+ $value =~ s/"/\\"/; >+ >+ my $term = '@attr ' . $_pqf_mapping->{$index} . ' "' . $value . '"'; >+ >+ if ( $query ) { >+ $query = '@and ' . $query . ' ' . $term; >+ } else { >+ $query = $term; >+ } >+ } >+ >+ return $query; >+} >+ >+sub _db_query_get_match_conditions { >+ my ( $index, $value ) = @_; >+ >+ if ( $value =~ /\*/ ) { >+ $value =~ s/\*/%/; >+ return { -like => $value }; >+ } elsif ( $_batch_db_text_columns->{$index} ) { >+ return map +{ -regexp => '[[:<:]]' . $_ . '[[:>:]]' }, split( /\s+/, $value ); >+ } else { >+ return $value; >+ } >+} >+ >+sub _batch_db_query_from_terms { >+ my ( $import_batch_id, $terms ) = @_; >+ >+ my @db_terms; >+ >+ while ( my ( $index, $value ) = each %$terms ) { >+ if ( $index eq 'keyword' ) { >+ my @term; >+ >+ while ( my ( $index, $db_column ) = each %$_batch_db_mapping ) { >+ push @term, { $db_column => [ _db_query_get_match_conditions( $index, $value ) ] }; >+ } >+ >+ # These are implicitly joined with OR because the arrayref doesn't start with -and >+ push @db_terms, \@term; >+ } elsif ( $_batch_db_mapping->{$index} ) { >+ push @db_terms, $_batch_db_mapping->{$index} => [ -and => _db_query_get_match_conditions( $index, $value ) ]; >+ } else { >+ # No such index, we should fail >+ return undef; >+ } >+ } >+ >+ return { >+ -and => [ >+ import_batch_id => $import_batch_id, >+ @db_terms >+ ], >+ }, >+} >+ > sub _start_worker { >- my ( $self, $server, $query ) = @_; >+ my ( $self, $server, $terms ) = @_; > pipe my $readfh, my $writefh; > > # Accessing the cache or Koha database after the fork is risky, so get any resources we need >@@ -207,6 +301,8 @@ sub _start_worker { > my $connection; > my ( $num_hits, $num_fetched, $hits, $results ); > >+ my $pqf_query = _pqf_query_from_terms( $terms ); >+ > eval { > if ( $server->{type} eq 'z3950' ) { > my $zoptions = ZOOM::Options->new(); >@@ -220,66 +316,64 @@ sub _start_worker { > $connection = ZOOM::Connection->create($zoptions); > > $connection->connect( $server->{host}, $server->{port} ); >- $results = $connection->search_pqf( $query ); # Starts the search >+ $results = $connection->search_pqf( $pqf_query ); # Starts the search > } elsif ( $server->{type} eq 'koha' ) { > $connection = C4::Context->Zconn( $server->{extra} ); >- $results = $connection->search_pqf( $query ); # Starts the search >+ $results = $connection->search_pqf( $pqf_query ); # Starts the search > } elsif ( $server->{type} eq 'batch' ) { > $server->{encoding} = 'utf-8'; > } > }; > if ($@) { > store_fd { >- error => $connection ? $connection->exception() : $@, >+ error => $connection ? $connection->exception()->message() : $@, > server => $server, > }, $writefh; > exit; > } > >+ my $error; > if ( $server->{type} eq 'batch' ) { >- # TODO: actually handle PQF >- $query =~ s/@\w+ (?:\d+=\d+ )?//g; >- $query =~ s/"//g; >- > my $schema = Koha::Database->new->schema; >- $schema->storage->debug(1); >- my $match_condition = [ map +{ -like => '%' . $_ . '%' }, split( /\s+/, $query ) ]; >- $hits = [ $schema->resultset('ImportRecord')->search( >- { >- import_batch_id => $server->{extra}, >- -or => [ >- { 'import_biblios.title' => $match_condition }, >- { 'import_biblios.author' => $match_condition }, >- { 'import_biblios.isbn' => $match_condition }, >- { 'import_biblios.issn' => $match_condition }, >- ], >- }, >- { >- join => [ qw( import_biblios ) ], >- rows => $self->{fetch}, >- } >- )->get_column( 'marc' )->all ]; >- >- $num_hits = $num_fetched = scalar @$hits; >+ my $db_query = _batch_db_query_from_terms( $server->{extra}, $terms ); >+ >+ if ( $db_query ) { >+ $hits = [ $schema->resultset('ImportRecord')->search( >+ $db_query, >+ { >+ columns => [ 'import_record_id', { record => 'marc' } ], >+ join => [ 'import_biblios' ], >+ offset => $self->{offset}, >+ result_class => 'DBIx::Class::ResultClass::HashRefInflator', >+ rows => $self->{fetch}, >+ } >+ )->all ]; >+ >+ $num_hits = $num_fetched = scalar @$hits; >+ } else { >+ $error = "Invalid search"; >+ } > } else { > $num_hits = $results->size; > $num_fetched = ( $self->{offset} + $self->{fetch} ) < $num_hits ? $self->{fetch} : $num_hits; > >- $hits = [ map { $_->raw() } @{ $results->records( $self->{offset}, $num_fetched, 1 ) } ]; >+ $hits = [ map +{ record => $_->raw() }, @{ $results->records( $self->{offset}, $num_fetched, 1 ) } ]; >+ >+ $error = $connection->exception()->message() if ( !@$hits && $connection && $connection->exception() ); > } > >- if ( !@$hits && $connection && $connection->exception() ) { >+ if ( $error ) { > store_fd { >- error => $connection->exception(), >+ error => $error, > server => $server, > }, $writefh; > exit; > } > > if ( $server->{type} eq 'koha' ) { >- $hits = [ map { C4::Search::new_record_from_zebra( $server->{extra}, $_ ) } @$hits ]; >+ $hits = [ map +{ %$_, record => C4::Search::new_record_from_zebra( $server->{extra}, $_->{record} ) }, @$hits ]; > } else { >- $hits = [ map { $self->_import_record( $_, $marcflavour, $server->{encoding} ? $server->{encoding} : "iso-5426" ) } @$hits ]; >+ $hits = [ map +{ %$_, record => $self->_import_record( $_->{record}, $marcflavour, $server->{encoding} ? $server->{encoding} : "iso-5426" ) }, @$hits ]; > } > > store_fd { >diff --git a/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js b/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js >index 0ace7ad..10a1975 100644 >--- a/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js >+++ b/koha-tmpl/intranet-tmpl/lib/koha/cateditor/search.js >@@ -22,42 +22,11 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { > var _records = {}; > var _last; > >- var _pqfMapping = { >- author: '1=1004', // s=al', >- cn_dewey: '1=13', >- cn_lc: '1=16', >- date: '1=30', // r=r', >- isbn: '1=7', >- issn: '1=8', >- lccn: '1=9', >- local_number: '1=12', >- music_identifier: '1=51', >- standard_identifier: '1=1007', >- subject: '1=21', // s=al', >- term: '1=1016', // t=l,r s=al', >- title: '1=4', // s=al', >- } >- > var Search = { > Init: function( options ) { > _options = options; > }, >- JoinTerms: function( terms ) { >- var q = ''; >- >- $.each( terms, function( i, term ) { >- var term = '@attr ' + _pqfMapping[ term[0] ] + ' "' + term[1].replace( '"', '\\"' ) + '"' >- >- if ( q ) { >- q = '@and ' + q + ' ' + term; >- } else { >- q = term; >- } >- } ); >- >- return q; >- }, >- Run: function( servers, q, options ) { >+ Run: function( servers, terms, options ) { > options = $.extend( { > offset: 0, > page_size: 20, >@@ -67,10 +36,16 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { > _records = {}; > _last = { > servers: servers, >- q: q, >+ terms: terms, > options: options, > }; > >+ var newTerms = {}; >+ $.each( terms, function( index, value ) { >+ newTerms[ "term-" + index ] = value; >+ } ); >+ terms = newTerms; >+ > var itemTag = KohaBackend.GetSubfieldForKohaField('items.itemnumber')[0]; > > $.each( servers, function ( id, info ) { >@@ -81,15 +56,14 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { > > $.get( > '/cgi-bin/koha/svc/cataloguing/metasearch', >- { >- q: q, >+ $.extend( { > servers: Search.includedServers.join( ',' ), > offset: options.offset, > page_size: options.page_size, > sort_direction: options.sort_direction, > sort_key: options.sort_key, > resultset: options.resultset, >- } >+ }, terms ) > ) > .done( function( data ) { > _last.options.resultset = data.resultset; >@@ -115,7 +89,7 @@ define( [ 'koha-backend', 'marc-record' ], function( KohaBackend, MARC ) { > Fetch: function( options ) { > if ( !_last ) return; > $.extend( _last.options, options ); >- return Search.Run( _last.servers, _last.q, _last.options ); >+ return Search.Run( _last.servers, _last.terms, _last.options ); > } > }; > >diff --git a/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc b/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc >index d1ab654..584f0c0 100644 >--- a/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc >+++ b/koha-tmpl/intranet-tmpl/prog/en/includes/cateditor-ui.inc >@@ -155,17 +155,17 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr > > $('#quicksearch .search-box').each( function() { > shortcut.add( 'enter', $.proxy( function() { >- var terms = []; >+ var terms = {}; > > $('#quicksearch .search-box').each( function() { > if ( !this.value ) return; > >- terms.push( [ $(this).data('qualifier'), this.value ] ); >+ terms[ $(this).data('qualifier') ] = this.value; > } ); > >- if ( !terms.length ) return; >+ if ( $.isEmptyObject(terms) ) return; > >- if ( Search.Run( z3950Servers, Search.JoinTerms(terms) ) ) { >+ if ( Search.Run( z3950Servers, terms ) ) { > $("#search-overlay").show(); > showResultsBox(); > } >@@ -455,17 +455,17 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr > } > > function startAdvancedSearch() { >- var terms = []; >+ var terms = {}; > > $('#advanced-search-ui .search-box').each( function() { > if ( !this.value ) return; > >- terms.push( [ $(this).data('qualifier'), this.value ] ); >+ terms[ $(this).data('qualifier') ] = this.value; > } ); > >- if ( !terms.length ) return; >+ if ( $.isEmptyObject(terms) ) return; > >- if ( Search.Run( z3950Servers, Search.JoinTerms(terms) ) ) { >+ if ( Search.Run( z3950Servers, terms ) ) { > $('#advanced-search-ui').modal('hide'); > $("#search-overlay").show(); > showResultsBox(); >@@ -538,19 +538,15 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr > $.each( data.hits, function( undef, hit ) { > backends.search.records[ hit.server + ':' + hit.index ] = hit.record; > >- switch ( hit.server ) { >- case 'koha:biblioserver': >- var bibnumField = hit.record.field( bibnumMap[0] ); >+ hit.id = 'search/' + hit.server + ':' + hit.index; >+ if ( hit.server == 'koha:biblioserver' ) { >+ var bibnumField = hit.record.field( bibnumMap[0] ); > >- if ( bibnumField && bibnumField.hasSubfield( bibnumMap[1] ) ) { >- hit.id = 'catalog/' + bibnumField.subfield( bibnumMap[1] ); >- break; >- } >- >- // Otherwise, fallthrough >- >- default: >- hit.id = 'search/' + hit.server + ':' + hit.index; >+ if ( bibnumField && bibnumField.hasSubfield( bibnumMap[1] ) ) { >+ hit.id = 'catalog/' + bibnumField.subfield( bibnumMap[1] ); >+ } >+ } else if ( /^batch:/.test(hit.server) && hit.import_record_id ) { >+ hit.id = hit.server + '/' + hit.import_record_id; > } > > var result = '<tr>'; >@@ -702,13 +698,6 @@ require( [ 'koha-backend', 'search', 'macros', 'marc-editor', 'marc-record', 'pr > if ( saved_val != null ) setSaveTargetChecked( target_id, saved_val ); > } ); > break; >- case 'selected_search_targets': >- $.each( z3950Servers, function( server_id, server ) { >- var saved_val = Preferences.user.selected_search_targets[server_id]; >- >- if ( saved_val != null ) server.checked = saved_val; >- } ); >- break; > } > } > >diff --git a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/editor.tt b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/editor.tt >index 8d8d89b..e6441cb 100644 >--- a/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/editor.tt >+++ b/koha-tmpl/intranet-tmpl/prog/en/modules/cataloguing/editor.tt >@@ -82,7 +82,7 @@ > <fieldset class="brief"> > <ol> > <li><label for="search-by-keywords">Keywords:</label></li> >- <li><input class="search-box" data-qualifier="term" id="search-by-keywords" placeholder="(Ctrl-Alt-K)" /></li> >+ <li><input class="search-box" data-qualifier="keyword" id="search-by-keywords" placeholder="(Ctrl-Alt-K)" /></li> > <li><label for="search-by-author">Author:</label></li> > <li><input class="search-box" data-qualifier="author" id="search-by-author" placeholder="(Ctrl-Alt-A)" /></li> > <li><label for="search-by-isbn">ISBN:</label></li> >@@ -151,7 +151,7 @@ > </li> > <li> > <label for="advanced-search-by-lc-number">LC call number:</label> >- <input class="search-box" data-qualifier="cn_lc" id="advanced-search-by-lc-number" /> >+ <input class="search-box" data-qualifier="cn-lc" id="advanced-search-by-lc-number" /> > </li> > <li> > <label for="advanced-search-by-lccn">LCCN:</label> >@@ -159,19 +159,19 @@ > </li> > <li> > <label for="advanced-search-by-control-number">Control number:</label> >- <input class="search-box" data-qualifier="local_number" id="advanced-search-by-control-number" /> >+ <input class="search-box" data-qualifier="local-number" id="advanced-search-by-control-number" /> > </li> > <li> > <label for="advanced-search-by-dewey">Dewey number:</label> >- <input class="search-box" data-qualifier="cn_dewey" id="advanced-search-by-dewey" /> >+ <input class="search-box" data-qualifier="cn-dewey" id="advanced-search-by-dewey" /> > </li> > <li> > <label for="advanced-search-by-standard-number">Standard number:</label> >- <input class="search-box" data-qualifier="standard_identifier" id="advanced-search-by-standard-number" /> >+ <input class="search-box" data-qualifier="standard-identifier" id="advanced-search-by-standard-number" /> > </li> > <li> > <label for="advanced-search-by-publisher-number">Publisher number:</label> >- <input class="search-box" data-qualifier="music_identifier" id="advanced-search-by-publisher-number" /> >+ <input class="search-box" data-qualifier="music-identifier" id="advanced-search-by-publisher-number" /> > </li> > <li> > <label for="advanced-search-by-publication-date">Publication date:</label> >diff --git a/svc/cataloguing/metasearch b/svc/cataloguing/metasearch >index 110732f..4e946d0 100755 >--- a/svc/cataloguing/metasearch >+++ b/svc/cataloguing/metasearch >@@ -25,7 +25,7 @@ use Koha::MetaSearcher; > > my ( $query, $response ) = C4::Service->init( catalogue => 1 ); > >-my ( $query_string, $servers ) = C4::Service->require_params( 'q', 'servers' ); >+my ( $servers ) = C4::Service->require_params( 'servers' ); > > my $server_errors = {}; > >@@ -35,19 +35,27 @@ my $offset = $query->param( 'offset' ) || 0; > my $page_size = $query->param( 'page_size' ) || 20; > my $fetched = $query->param( 'fetched' ) || 100; > >+my $terms = {}; >+ >+foreach my $term ( $query->param ) { >+ next unless ( $term =~ /^term-([A-Za-z0-9-]+)/ ); >+ >+ $terms->{$1} = $query->param($term); >+} >+ > my $searcher = Koha::MetaSearcher->new( { > fetched => $fetched, > on_error => sub { > my ( $server, $exception ) = @_; > >- $server_errors->{ $server->{id} } = $exception->message; >+ $server_errors->{ $server->{id} } = $exception; > }, > } ); > > $searcher->resultset( $query->param('resultset') ) if ( $query->param('resultset') ); > > my @server_ids = split( /,/, $servers ); >-my $stats = $searcher->search( \@server_ids, $query_string ); >+my $stats = $searcher->search( \@server_ids, $terms ); > > $searcher->sort( $sort_key, $sort_direction eq 'desc' ? -1 : 1 ); > >@@ -55,10 +63,11 @@ my @hits; > > foreach my $hit ( $searcher->results( $offset, $page_size ) ) { > push @hits, { >+ %$hit, > server => $hit->{server}->{id}, > index => $hit->{index}, > record => $hit->{record}->as_xml_record(), >- metadata => $hit->{metadata} >+ metadata => $hit->{metadata}, > }; > } > >-- >2.7.4
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Actions:
View
|
Diff
|
Splinter Review
Attachments on
bug 18823
:
64434
|
66882
|
66883
|
66884
|
66885
|
66919
|
77240
|
77241
|
77242
|
77243
|
77244
|
79240
|
79241
|
79242
|
79243
|
79244
|
80519
|
80536
|
80604
|
80606
|
80608
|
80613
|
80626
|
80627
|
80628
|
80637
|
80639
|
80651
|
80861
|
80866
|
80868
|
80869
|
80870
|
80871
|
80872
|
80873
|
80947
|
85355
|
85356
|
85357
|
85358
|
85359
|
85376
|
85719
|
85720
|
85785
|
85828
|
86087
|
86173
|
86174
|
86175
|
86176
|
86177
|
86178
|
86179
|
86180
|
86181
|
86182
|
86183
|
86791
|
86792
|
86793
|
86794
|
86795
|
86796
|
86797
|
86798
|
86799
|
88417
|
88418
|
88419
|
88420
|
88421
|
88422
|
88423
|
88424
|
88425
|
88426
|
88427
|
88428
|
88429
|
88430
|
88431
|
88432
|
88493
|
88494
|
88495
|
88496
|
88497
|
88498
|
88499
|
88500
|
88501
|
88502
|
88503
|
88504
|
88505
|
88506
|
88507
|
88508
|
91205
|
91206
|
91207
|
91208
|
91209
|
91210
|
91211
|
91212
|
91213
|
91214
|
91215
|
91216
|
91217
|
91218
|
91219
|
92051
|
92120
|
92124
|
94436
|
94437
|
94438
|
94439
|
94440
|
94441
|
94442
|
94443
|
94444
|
94445
|
94446
|
94447
|
94448
|
94449
|
94450
|
94451
|
94452
|
99092
|
99093
|
99094
|
99095
|
99096
|
99097
|
99098
|
99099
|
99100
|
99101
|
99102
|
99103
|
99104
|
99105
|
99106
|
99107
|
99108
|
99109