Mercurial Hosting > luan
comparison lucene/src/luan/modules/lucene/LuceneIndex.java @ 545:ddcd4296107a
clean up lucene search
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 14 Jun 2015 01:34:42 -0600 |
parents | c5a93767cc5c |
children | eaef1005ab87 |
comparison
equal
deleted
inserted
replaced
544:c5a93767cc5c | 545:ddcd4296107a |
---|---|
16 import org.apache.lucene.index.IndexWriter; | 16 import org.apache.lucene.index.IndexWriter; |
17 import org.apache.lucene.index.IndexWriterConfig; | 17 import org.apache.lucene.index.IndexWriterConfig; |
18 import org.apache.lucene.index.DirectoryReader; | 18 import org.apache.lucene.index.DirectoryReader; |
19 import org.apache.lucene.index.Term; | 19 import org.apache.lucene.index.Term; |
20 import org.apache.lucene.index.SnapshotDeletionPolicy; | 20 import org.apache.lucene.index.SnapshotDeletionPolicy; |
21 import org.apache.lucene.index.AtomicReaderContext; | |
21 import org.apache.lucene.store.Directory; | 22 import org.apache.lucene.store.Directory; |
22 import org.apache.lucene.store.FSDirectory; | 23 import org.apache.lucene.store.FSDirectory; |
23 import org.apache.lucene.util.Version; | 24 import org.apache.lucene.util.Version; |
24 import org.apache.lucene.search.Query; | 25 import org.apache.lucene.search.Query; |
25 import org.apache.lucene.search.TermQuery; | 26 import org.apache.lucene.search.TermQuery; |
26 import org.apache.lucene.search.TopDocs; | 27 import org.apache.lucene.search.TopDocs; |
27 import org.apache.lucene.search.Sort; | 28 import org.apache.lucene.search.Sort; |
28 import org.apache.lucene.search.SortField; | 29 import org.apache.lucene.search.SortField; |
29 import org.apache.lucene.search.IndexSearcher; | 30 import org.apache.lucene.search.IndexSearcher; |
31 import org.apache.lucene.search.TotalHitCountCollector; | |
32 import org.apache.lucene.search.ScoreDoc; | |
33 import org.apache.lucene.search.Collector; | |
34 import org.apache.lucene.search.Scorer; | |
30 import sane.lucene.queryparser.SaneQueryParser; | 35 import sane.lucene.queryparser.SaneQueryParser; |
31 import sane.lucene.queryparser.FieldParser; | 36 import sane.lucene.queryparser.FieldParser; |
32 import sane.lucene.queryparser.MultiFieldParser; | 37 import sane.lucene.queryparser.MultiFieldParser; |
33 import sane.lucene.queryparser.StringFieldParser; | 38 import sane.lucene.queryparser.StringFieldParser; |
34 import sane.lucene.queryparser.NumberFieldParser; | 39 import sane.lucene.queryparser.NumberFieldParser; |
39 import luan.LuanTable; | 44 import luan.LuanTable; |
40 import luan.LuanFunction; | 45 import luan.LuanFunction; |
41 import luan.LuanJavaFunction; | 46 import luan.LuanJavaFunction; |
42 import luan.LuanException; | 47 import luan.LuanException; |
43 import luan.LuanMeta; | 48 import luan.LuanMeta; |
49 import luan.LuanRuntimeException; | |
44 import org.slf4j.Logger; | 50 import org.slf4j.Logger; |
45 import org.slf4j.LoggerFactory; | 51 import org.slf4j.LoggerFactory; |
46 | 52 |
47 | 53 |
48 public final class LuceneIndex implements Closeable { | 54 public final class LuceneIndex implements Closeable { |
50 | 56 |
51 private static final String FLD_NEXT_ID = "nextId"; | 57 private static final String FLD_NEXT_ID = "nextId"; |
52 private static final Analyzer analyzer = new KeywordAnalyzer(); | 58 private static final Analyzer analyzer = new KeywordAnalyzer(); |
53 public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer); | 59 public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer); |
54 | 60 |
61 final LuanTable myTable; | |
55 final Lock writeLock = new ReentrantLock(); | 62 final Lock writeLock = new ReentrantLock(); |
56 private final File indexDir; | 63 private final File indexDir; |
57 final SnapshotDeletionPolicy snapshotDeletionPolicy; | 64 final SnapshotDeletionPolicy snapshotDeletionPolicy; |
58 final IndexWriter writer; | 65 final IndexWriter writer; |
59 private DirectoryReader reader; | 66 private DirectoryReader reader; |
60 private LuceneSearcher searcher; | 67 private IndexSearcher searcher; |
68 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); | |
61 private boolean isClosed = false; | 69 private boolean isClosed = false; |
62 private final MultiFieldParser mfp = new MultiFieldParser(); | 70 private final MultiFieldParser mfp = new MultiFieldParser(); |
63 | 71 |
64 public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException { | 72 public LuceneIndex(LuanState luan,String indexDirStr,LuanTable myTable) throws LuanException, IOException { |
73 this.myTable = myTable; | |
65 mfp.fields.put( "type", STRING_FIELD_PARSER ); | 74 mfp.fields.put( "type", STRING_FIELD_PARSER ); |
66 mfp.fields.put( "id", NumberFieldParser.LONG ); | 75 mfp.fields.put( "id", NumberFieldParser.LONG ); |
67 File indexDir = new File(indexDirStr); | 76 File indexDir = new File(indexDirStr); |
68 this.indexDir = indexDir; | 77 this.indexDir = indexDir; |
69 Directory dir = FSDirectory.open(indexDir); | 78 Directory dir = FSDirectory.open(indexDir); |
73 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); | 82 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); |
74 writer = new IndexWriter(dir,conf); | 83 writer = new IndexWriter(dir,conf); |
75 writer.commit(); // commit index creation | 84 writer.commit(); // commit index creation |
76 reader = DirectoryReader.open(dir); | 85 reader = DirectoryReader.open(dir); |
77 luan.onClose(this); | 86 luan.onClose(this); |
78 searcher = new LuceneSearcher(this,reader); | 87 searcher = new IndexSearcher(reader); |
79 initId(luan); | 88 initId(luan); |
80 } | 89 } |
81 | 90 |
82 Document toLucene(LuanState luan,LuanTable table) throws LuanException { | 91 Document toLucene(LuanState luan,LuanTable table) throws LuanException { |
83 return LuceneDocument.toLucene(luan,table,mfp.fields.keySet()); | 92 return LuceneDocument.toLucene(luan,table,mfp.fields.keySet()); |
85 | 94 |
86 public LuceneWriter openWriter() { | 95 public LuceneWriter openWriter() { |
87 return new LuceneWriter(this); | 96 return new LuceneWriter(this); |
88 } | 97 } |
89 | 98 |
90 synchronized LuceneSearcher openSearcher() throws IOException { | 99 private synchronized IndexSearcher openSearcher() throws IOException { |
91 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); | 100 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); |
92 if( newReader != null ) { | 101 if( newReader != null ) { |
93 reader.decRef(); | 102 reader.decRef(); |
94 reader = newReader; | 103 reader = newReader; |
95 searcher = new LuceneSearcher(this,reader); | 104 searcher = new IndexSearcher(reader); |
96 } | 105 } |
97 reader.incRef(); | 106 reader.incRef(); |
98 return searcher; | 107 return searcher; |
108 } | |
109 | |
110 // call in finally block | |
111 private static void close(IndexSearcher searcher) throws IOException { | |
112 searcher.getIndexReader().decRef(); | |
99 } | 113 } |
100 | 114 |
101 LuceneSnapshot openSnapshot() throws IOException { | 115 LuceneSnapshot openSnapshot() throws IOException { |
102 return new LuceneSnapshot(this); | 116 return new LuceneSnapshot(this); |
103 } | 117 } |
117 private long id = 0; | 131 private long id = 0; |
118 private long idLim = 0; | 132 private long idLim = 0; |
119 private final int idBatch = 10; | 133 private final int idBatch = 10; |
120 | 134 |
121 private void initId(LuanState luan) throws LuanException, IOException { | 135 private void initId(LuanState luan) throws LuanException, IOException { |
122 IndexSearcher searcher = this.searcher.searcher; | |
123 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); | 136 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); |
124 switch(td.totalHits) { | 137 switch(td.totalHits) { |
125 case 0: | 138 case 0: |
126 break; // do nothing | 139 break; // do nothing |
127 case 1: | 140 case 1: |
170 | 183 |
171 public String to_string() { | 184 public String to_string() { |
172 return writer.getDirectory().toString(); | 185 return writer.getDirectory().toString(); |
173 } | 186 } |
174 | 187 |
175 public void Writer(LuanState luan,LuanFunction fn) throws LuanException, IOException { | 188 public void writer(LuanState luan,LuanFunction fn) throws LuanException, IOException { |
176 LuceneWriter writer = openWriter(); | 189 LuceneWriter writer = openWriter(); |
177 try { | 190 try { |
178 luan.call( fn, new Object[]{writer.table()} ); | 191 luan.call( fn, new Object[]{writer.table()} ); |
179 writer.commit(); | 192 writer.commit(); |
180 } finally { | 193 } finally { |
181 writer.close(); | 194 writer.close(); |
182 } | |
183 } | |
184 | |
185 public Object Searcher(LuanState luan,LuanFunction fn) throws LuanException, IOException { | |
186 LuceneSearcher searcher = openSearcher(); | |
187 try { | |
188 return luan.call( fn, new Object[]{searcher.table()} ); | |
189 } finally { | |
190 searcher.close(); | |
191 } | 195 } |
192 } | 196 } |
193 | 197 |
194 public void close() throws IOException { | 198 public void close() throws IOException { |
195 if( !isClosed ) { | 199 if( !isClosed ) { |
203 if( !isClosed ) { | 207 if( !isClosed ) { |
204 logger.error("not closed"); | 208 logger.error("not closed"); |
205 close(); | 209 close(); |
206 } | 210 } |
207 super.finalize(); | 211 super.finalize(); |
212 } | |
213 | |
214 | |
215 | |
216 private static class DocFn extends LuanFunction { | |
217 final IndexSearcher searcher; | |
218 int docID; | |
219 | |
220 DocFn(IndexSearcher searcher) { | |
221 this.searcher = searcher; | |
222 } | |
223 | |
224 @Override public Object call(LuanState luan,Object[] args) throws LuanException { | |
225 try { | |
226 return LuceneDocument.toTable(luan,searcher.doc(docID)); | |
227 } catch(IOException e) { | |
228 throw luan.exception(e); | |
229 } | |
230 } | |
231 } | |
232 | |
233 private static abstract class MyCollector extends Collector { | |
234 int docBase; | |
235 int i = 0; | |
236 | |
237 @Override public void setScorer(Scorer scorer) {} | |
238 @Override public void setNextReader(AtomicReaderContext context) { | |
239 this.docBase = context.docBase; | |
240 } | |
241 @Override public boolean acceptsDocsOutOfOrder() { | |
242 return true; | |
243 } | |
244 } | |
245 | |
246 public int advanced_search( final LuanState luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException { | |
247 Utils.checkNotNull(luan,queryStr); | |
248 Query query = parseQuery(queryStr); | |
249 IndexSearcher searcher = threadLocalSearcher.get(); | |
250 boolean inTransaction = searcher != null; | |
251 if( !inTransaction ) | |
252 searcher = openSearcher(); | |
253 try { | |
254 if( fn!=null && n==null ) { | |
255 if( sortStr != null ) | |
256 throw luan.exception("sort must be nil when n is nil"); | |
257 final DocFn docFn = new DocFn(searcher); | |
258 MyCollector col = new MyCollector() { | |
259 @Override public void collect(int doc) { | |
260 try { | |
261 docFn.docID = doc; | |
262 luan.call(fn,new Object[]{++i,docFn}); | |
263 } catch(LuanException e) { | |
264 throw new LuanRuntimeException(e); | |
265 } | |
266 } | |
267 }; | |
268 try { | |
269 searcher.search(query,col); | |
270 } catch(LuanRuntimeException e) { | |
271 throw (LuanException)e.getCause(); | |
272 } | |
273 return col.i; | |
274 } | |
275 if( fn==null || n==0 ) { | |
276 TotalHitCountCollector thcc = new TotalHitCountCollector(); | |
277 searcher.search(query,thcc); | |
278 return thcc.getTotalHits(); | |
279 } | |
280 Sort sort = sortStr==null ? null : parseSort(sortStr); | |
281 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); | |
282 final ScoreDoc[] scoreDocs = td.scoreDocs; | |
283 DocFn docFn = new DocFn(searcher); | |
284 for( int i=0; i<scoreDocs.length; i++ ) { | |
285 docFn.docID = scoreDocs[i].doc; | |
286 luan.call(fn,new Object[]{i+1,docFn}); | |
287 } | |
288 return td.totalHits; | |
289 } finally { | |
290 if( !inTransaction ) | |
291 close(searcher); | |
292 } | |
293 } | |
294 | |
295 public Object search_in_transaction(LuanState luan,LuanFunction fn) throws LuanException, IOException { | |
296 if( threadLocalSearcher.get() != null ) | |
297 throw luan.exception("can't nest search_in_transaction calls"); | |
298 IndexSearcher searcher = openSearcher(); | |
299 threadLocalSearcher.set(searcher); | |
300 try { | |
301 return luan.call(fn); | |
302 } finally { | |
303 threadLocalSearcher.set(null); | |
304 close(searcher); | |
305 } | |
208 } | 306 } |
209 | 307 |
210 | 308 |
211 | 309 |
212 public final LuanMeta indexedFieldsMeta = new LuanMeta() { | 310 public final LuanMeta indexedFieldsMeta = new LuanMeta() { |