View | Details | Raw Unified | Return to bug 2539
Collapse All | Expand All

(-)a/misc/spellcheck_suggest/make_spellcheck_suggest.pl (-241 lines)
Lines 1-240 Link Here
1
#!/usr/bin/perl
2
## This Script creates a Koha suggest and spellcheck database
3
## for those features as visible on LibLime's opac: opac.liblime.com
4
## It also contains the needed specs for creating a table of
5
## queries for statistical purposes as well as a method of
6
## returning popular searches via the suggest and spellcheck.
7
## The code for running suggest and spellcheck can be found
8
## either in Koha 2.4 CVS (HEAD as of this writing) or at 
9
## LibLime's website in the downlaods
10
## section: http://liblime.com/c/downloads.html
11
##
12
##Author: Joshua Ferraro jmf at liblime dot com
13
##
14
## TODO: add suggest features, merge the two of them?
15
## There are a few configurable variables.  
16
17
use warnings;
18
19
## CONFIGURABLE VARIABLES ####################
20
##
21
 # These are the tags that have meaningful data
22
 # for the databases I've worked with (MARC21 only)
23
 # you may need to change them depending on your data
24
my @tags=(
25
#Tag documentation from http://lcweb.loc.gov/marc/bibliographic/ecbdhome.html
26
"020a", # INTERNATIONAL STANDARD BOOK NUMBER
27
#"022a", # INTERNATIONAL STANDARD SERIAL NUMBER
28
"100a", # MAIN ENTRY--PERSONAL NAME
29
"110a", # MAIN ENTRY--CORPORATE NAME
30
#"110b", #   Subordinate unit
31
#"110c", #   Location of meeting
32
#"111a", # MAIN ENTRY--MEETING NAME
33
#"111c", #   Location of meeting
34
"130a", # MAIN ENTRY--UNIFORM TITLE
35
"240a", # UNIFORM TITLE
36
"245a", # TITLE STATEMENT
37
"245b", #   Remainder of title
38
"245c", #   Statement of responsibility, etc.
39
"245p", #   Name of part/section of a work
40
"246a", # VARYING FORM OF TITLE
41
"246b", #   Remainder of title
42
#"260b", # PUBLICATION, DISTRIBUTION, ETC. (IMPRINT)
43
"440a", # SERIES STATEMENT/ADDED ENTRY--TITLE
44
"440p", #   Name of part/section of a work
45
#"500a", # GENERAL NOTE
46
"505t", # FORMATTED CONTENTS NOTE (t is Title)
47
"511a", # PARTICIPANT OR PERFORMER NOTE
48
#"520a", # SUMMARY, ETC.
49
"534a", # ORIGINAL VERSION NOTE
50
#"534k", #   Key title of original
51
#"534t", #   Title statement of original
52
#"586a", # AWARDS NOTE
53
"600a", # SUBJECT ADDED ENTRY--PERSONAL NAME
54
"610a", # SUBJECT ADDED ENTRY--CORPORATE NAME
55
"611a", # SUBJECT ADDED ENTRY--MEETING NAME
56
"630a", # SUBJECT ADDED ENTRY--UNIFORM TITLE
57
"650a", # SUBJECT ADDED ENTRY--TOPICAL TERM
58
"651a", # SUBJECT ADDED ENTRY--GEOGRAPHIC NAME
59
"700a", # ADDED ENTRY--PERSONAL NAME
60
"710a", # ADDED ENTRY--CORPORATE NAME
61
#"711a", # ADDED ENTRY--MEETING NAME
62
#"720a", # ADDED ENTRY--UNCONTROLLED NAME
63
"730a", # ADDED ENTRY--UNIFORM TITLE
64
"740a", # ADDED ENTRY--UNCONTROLLED RELATED/ANALYTICAL TITLE
65
#"752a", # ADDED ENTRY--HIERARCHICAL PLACE NAME
66
"800a", # SERIES ADDED ENTRY--PERSONAL NAME
67
#"810a", # SERIES ADDED ENTRY--CORPORATE NAME
68
#"811a", # SERIES ADDED ENTRY--MEETING NAME
69
"830a", # SERIES ADDED ENTRY--UNIFORM TITLE
70
#"942k"  # Holdings Branch ?? Unique to NPL??
71
);
72
## Leave this next bit alone
73
use strict;
74
BEGIN {
75
    # find Koha's Perl modules
76
    # test carefully before changing this
77
    use FindBin;
78
    eval { require "$FindBin::Bin/../kohalib.pl" };
79
}
80
use C4::Context;
81
##
82
 # SUGGEST DATABASE INFO
83
 # You'll need to change this if you want to keep your 'suggest' database
84
 # separate from your Koha database -- simply comment out the next line
85
 # and uncomment the one after it, adding your site info (check out GRANT
86
 # syntax in the mysql manual if you're unsure how enable authentication)
87
#
88
my $dbh2 = C4::Context->dbh;
89
#
90
#my $dbh2=DBI->connect("DBI:mysql:<add your database name here>:localhost","<add your mysql user here>","<add your password here>");
91
########################################################################
92
## End of most common configurable variables: in most cases you won't need
93
## edit any further ... of course feel free to indulge yourself ;-)
94
########################################################################
95
my $dbh=C4::Context->dbh;
96
my $counter = 0;
97
98
# Check for existance of suggest database and add if it doesn't.
99
print "Step 1 of 5: Checking to make sure suggest tables exist\n";
100
my $check_tables_query = "select distinct resultcount from ?";
101
my @tables = ("notdistinctspchk", "notdistinctsugg", "spellcheck", "suggestions");
102
foreach my $table (@tables) {
103
  my $sth_check=$dbh2->prepare($check_tables_query) || die "cant prepare query: $DBI::errstr";
104
  my $rv = $sth_check->execute($table);
105
  if(!defined($rv)) {
106
    print "$table missing ... creating it now\n";
107
    my $create_this = "CREATE TABLE \'$table\' \(
108
  			display varchar\(40\) NOT NULL default \'\',
109
  			suggestion varchar\(40\) NOT NULL default \'\',
110
  			resultcount varchar\(40\) NOT NULL default \'0\'
111
			\) TYPE=MyISAM";
112
    my $sth_create = $dbh->prepare($create_this) || die "can't prepare query: $DBI::errstr";
113
    $sth_create->execute() || die "can't execute: $DBI::errstr";
114
    print "$table created ...\n";
115
  }else {
116
    print "$table exists ...  moving along\n";
117
  }
118
}
119
print "All tables present ... moving along\n";
120
121
print "Step 2 of 5: Deleting old data\n";
122
my $clear_out = "DELETE FROM notdistinctspchk";
123
# Clear out old data
124
my $sth_clear_out=$dbh2->prepare($clear_out) || die "cant prepare query";
125
$sth_clear_out->execute();
126
print "Step 3 of 5: Creating non-distinct table from various Koha tables\n";
127
my $query_words = "SELECT DISTINCT word, COUNT(word) FROM marc_word";
128
my $query_marc_subfields = "SELECT DISTINCT subfieldvalue, COUNT(subfieldvalue) FROM marc_subfield_table";
129
my $query_titles = "SELECT DISTINCT title, COUNT(title) FROM biblio GROUP BY title";
130
my $query_authors = "SELECT DISTINCT author, COUNT(author) FROM biblio GROUP BY author";
131
132
my @queries = ("$query_words", "$query_marc_subfields", "$query_titles", "$query_authors");
133
134
foreach my $query (@queries) {
135
	
136
	#we need to do some special stuff for marc_word and marc_subfield_table queries
137
	if ($query eq $queries[0]) { #marc_word
138
	my $listoftagsubfields;
139
	  my $notfirst;
140
	    foreach my $tag (@tags) {	
141
	      $listoftagsubfields.="$tag, ";
142
	      if (!$notfirst) {
143
	        $query.=" WHERE tagsubfield=\'$tag\'";
144
	        $notfirst = 1;
145
	      } else {
146
	        $query.=" OR tagsubfield=\'$tag\'";
147
	      }
148
	    }#foreach
149
	$query.=" GROUP BY word";
150
	print "Finished building marc_word list\n";
151
	print "Adding marc_word entries with the following tagsubfields:"."$listoftagsubfields"."\n";
152
	}
153
154
	if ($query eq $queries[1]) { #marc_subfield_table
155
	my $listofsubfieldstuff; #for testing
156
	my $notfirst;
157
          foreach my $tag (@tags) {
158
	    my $justtag = $tag;
159
	    $justtag =~ s/\D\Z//;
160
	    my $subfieldcode = $&;
161
	    $listofsubfieldstuff.="$justtag, "."$subfieldcode, ";
162
            if (!$notfirst) {
163
              $query.=" WHERE (tag=\'$justtag\' and subfieldcode=\'$subfieldcode\')";
164
	      $notfirst = 1;
165
            } else {
166
              $query.=" OR (tag=\'$justtag\' and subfieldcode=\'$subfieldcode\')";
167
            }
168
	  }#foreach
169
        $query.=" GROUP BY subfieldvalue";
170
	print "Finished building marc_subfield_table list\n";
171
	print "Adding marc_subfield_table entries with the following tags and subfields:"."$listofsubfieldstuff"."\n";
172
        }
173
174
	my $sth=$dbh->prepare($query) || die "cant prepare query";
175
	$sth->execute();
176
177
	my $insert = "INSERT INTO notdistinctspchk(suggestion,display,resultcount) VALUES(?,?,?)";
178
179
	my $sth2=$dbh2->prepare($insert);
180
181
	while (my ($phraseterm,$count)=$sth->fetchrow_array) {
182
		if ($phraseterm) {	
183
		  #$display looks exactly like the DB
184
		  my $display = $phraseterm;
185
		  #except for a few things
186
		  $display =~s/  / /g;
187
		  $display =~ s/^\s+//; #remove leading whitespace
188
		  $display  =~ s/\s+$//; #remove trailing whitespace
189
       		  $display =~ s/(\.|\/)/ /g;
190
191
		  #suggestion is tweaked for optimal searching
192
		  my $suggestion = $phraseterm;
193
		  $suggestion =~ tr/A-Z/a-z/;
194
		  $suggestion =~ s/(\.|\?|\:|\!|\'|,|\-|\"|\(|\)|\[|\]|\{|\})/ /g;
195
		  $suggestion =~s/(\Aand-or |\Aand\/or |\Aanon |\Aan |\Aa |\Abut |\Aby |\Ade |\Ader |\Adr |\Adu|et |\Afor |\Afrom |\Ain |\Ainto |\Ait |\Amy |\Anot |\Aon |\Aor |\Aper |\Apt |\Aspp |\Ato |\Avs |\Awith |\Athe )/ /g;
196
		  $suggestion =~s/( and-or | and\/or | anon | an | a | but | by | de | der | dr | du|et | for | from | in | into | it | my | not | on | or | per | pt | spp | to | vs | with | the )/ /g;
197
198
		  $suggestion =~s/  / /g;
199
200
		  $suggestion =~ s/^\s+//; #remove leading whitespace
201
		  $suggestion =~ s/\s+$//; #remove trailing whitespace
202
        
203
		  if (length($suggestion)>2) {
204
			$sth2->execute($suggestion,$display,$count) || die "can't execute write";
205
			$counter++;
206
		  } #if 
207
		} #if
208
	}#while
209
print $counter." more records added...\n";
210
$sth2->finish;
211
$sth->finish;
212
}
213
214
# Now grab distincts from there and insert into our REAL database
215
216
print "Step 4 of 5: Deleting old distinct entries\n";
217
my $clear_distincts = "DELETE FROM spellcheck";
218
219
# Clear out old data
220
my $sth_clear_distincts=$dbh2->prepare($clear_distincts) || die "cant prepare query";
221
$sth_clear_distincts->execute();
222
223
print "Step 5 of 5: Creating distinct spellcheck table out of non-distinct table\n";
224
my $query_distincts = "SELECT DISTINCT suggestion, display, COUNT(display) FROM notdistinctspchk GROUP BY suggestion";
225
my $insert_distincts = "INSERT INTO spellcheck(suggestion,display,resultcount) VALUES(?,?,?)";
226
my $distinctcounter = 0;
227
228
my $sth=$dbh2->prepare($query_distincts) || die "cant prepare query";
229
$sth->execute();
230
my $sth2=$dbh2->prepare($insert_distincts) || die "cant prepare query";
231
while (my ($suggestion,$display,$count)=$sth->fetchrow_array) {
232
	if ($count) {
233
		$sth2->execute($suggestion,$display,$count) || die "can't execute write";
234
		$distinctcounter++;
235
	}
236
}
237
print "Finished: total distinct items added to spellcheck: "."$distinctcounter\n";
238
239
$dbh->disconnect();
240
$dbh2->disconnect();
241
- 

Return to bug 2539