Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1333:25746915a241

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge Luan and LuanState
author Franklin Schmidt <fschmidt@gmail.com>
date Tue, 12 Feb 2019 22:33:40 -0700
parents 48f302bdc187
children e0cf0d108a77
comparison
equal deleted inserted replaced
1332:11b7e11f9ed5 1333:25746915a241
67 import luan.modules.lucene.queryparser.StringFieldParser; 67 import luan.modules.lucene.queryparser.StringFieldParser;
68 import luan.modules.lucene.queryparser.NumberFieldParser; 68 import luan.modules.lucene.queryparser.NumberFieldParser;
69 import luan.lib.parser.ParseException; 69 import luan.lib.parser.ParseException;
70 import luan.modules.Utils; 70 import luan.modules.Utils;
71 import luan.Luan; 71 import luan.Luan;
72 import luan.LuanState;
73 import luan.LuanTable; 72 import luan.LuanTable;
74 import luan.LuanFunction; 73 import luan.LuanFunction;
75 import luan.LuanCloner; 74 import luan.LuanCloner;
76 import luan.LuanException; 75 import luan.LuanException;
77 import luan.LuanRuntimeException; 76 import luan.LuanRuntimeException;
101 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); 100 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
102 private File fileDir; 101 private File fileDir;
103 private int writeCount; 102 private int writeCount;
104 103
105 private final ConcurrentMap<String,Map<String,LuanFunction>> indexedOnlyFields = new ConcurrentHashMap<String,Map<String,LuanFunction>>(); 104 private final ConcurrentMap<String,Map<String,LuanFunction>> indexedOnlyFields = new ConcurrentHashMap<String,Map<String,LuanFunction>>();
106 private final LuanState luanMine = new LuanState(); 105 private final Luan luanMine = new Luan();
107 private final LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE); 106 private final LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE);
108 107
109 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) 108 public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
110 throws LuanException, IOException 109 throws LuanException, IOException
111 { 110 {
112 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 111 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
113 mfp.fields.put( "type", STRING_FIELD_PARSER ); 112 mfp.fields.put( "type", STRING_FIELD_PARSER );
114 mfp.fields.put( "id", NumberFieldParser.LONG ); 113 mfp.fields.put( "id", NumberFieldParser.LONG );
165 BytesRef br = new BytesRef(); 164 BytesRef br = new BytesRef();
166 NumericUtils.longToPrefixCoded(value,0,br); 165 NumericUtils.longToPrefixCoded(value,0,br);
167 return new Term(key,br); 166 return new Term(key,br);
168 } 167 }
169 168
170 public void delete(LuanState luan,String queryStr) throws LuanException, IOException, ParseException { 169 public void delete(Luan luan,String queryStr) throws LuanException, IOException, ParseException {
171 Query query = SaneQueryParser.parseQuery(mfp,queryStr); 170 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
172 171
173 boolean commit = !writeLock.isHeldByCurrentThread(); 172 boolean commit = !writeLock.isHeldByCurrentThread();
174 writeLock.lock(); 173 writeLock.lock();
175 try { 174 try {
186 indexedOnlyFields.putIfAbsent(type,new ConcurrentHashMap<String,LuanFunction>()); 185 indexedOnlyFields.putIfAbsent(type,new ConcurrentHashMap<String,LuanFunction>());
187 Map<String,LuanFunction> map = indexedOnlyFields.get(type); 186 Map<String,LuanFunction> map = indexedOnlyFields.get(type);
188 map.put(field,fn); 187 map.put(field,fn);
189 } 188 }
190 189
191 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException { 190 public void save(Luan luan,LuanTable doc) throws LuanException, IOException {
192 Set indexedOnlySet = new HashSet(); 191 Set indexedOnlySet = new HashSet();
193 Object typeObj = doc.get("type"); 192 Object typeObj = doc.get("type");
194 if( typeObj==null ) 193 if( typeObj==null )
195 throw new LuanException("missing 'type' field"); 194 throw new LuanException("missing 'type' field");
196 if( !(typeObj instanceof String) ) 195 if( !(typeObj instanceof String) )
229 wrote(); 228 wrote();
230 writeLock.unlock(); 229 writeLock.unlock();
231 } 230 }
232 } 231 }
233 232
234 public void update_in_transaction(LuanState luan,LuanFunction fn) throws IOException, LuanException { 233 public void update_in_transaction(Luan luan,LuanFunction fn) throws IOException, LuanException {
235 boolean commit = !writeLock.isHeldByCurrentThread(); 234 boolean commit = !writeLock.isHeldByCurrentThread();
236 writeLock.lock(); 235 writeLock.lock();
237 try { 236 try {
238 fn.call(luan); 237 fn.call(luan);
239 if(commit) writer.commit(); 238 if(commit) writer.commit();
241 wrote(); 240 wrote();
242 writeLock.unlock(); 241 writeLock.unlock();
243 } 242 }
244 } 243 }
245 244
246 public void run_in_lock(LuanState luan,LuanFunction fn) throws IOException, LuanException { 245 public void run_in_lock(Luan luan,LuanFunction fn) throws IOException, LuanException {
247 if( writeLock.isHeldByCurrentThread() ) 246 if( writeLock.isHeldByCurrentThread() )
248 throw new RuntimeException(); 247 throw new RuntimeException();
249 writeLock.lock(); 248 writeLock.lock();
250 try { 249 try {
251 synchronized(this) { 250 synchronized(this) {
276 default: 275 default:
277 throw new RuntimeException(); 276 throw new RuntimeException();
278 } 277 }
279 } 278 }
280 279
281 public synchronized long nextId(LuanState luan) throws LuanException, IOException { 280 public synchronized long nextId(Luan luan) throws LuanException, IOException {
282 if( ++id > idLim ) { 281 if( ++id > idLim ) {
283 idLim += idBatch; 282 idLim += idBatch;
284 LuanTable doc = new LuanTable(luan); 283 LuanTable doc = new LuanTable(luan);
285 doc.rawPut( "type", "next_id" ); 284 doc.rawPut( "type", "next_id" );
286 doc.rawPut( FLD_NEXT_ID, idLim ); 285 doc.rawPut( FLD_NEXT_ID, idLim );
312 */ 311 */
313 public SnapshotDeletionPolicy snapshotDeletionPolicy() { 312 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
314 return snapshotDeletionPolicy; 313 return snapshotDeletionPolicy;
315 } 314 }
316 315
317 public Object snapshot(LuanState luan,LuanFunction fn) throws LuanException, IOException { 316 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException {
318 IndexCommit ic = snapshotDeletionPolicy.snapshot(); 317 IndexCommit ic = snapshotDeletionPolicy.snapshot();
319 try { 318 try {
320 String dir = fileDir.toString(); 319 String dir = fileDir.toString();
321 LuanTable fileNames = new LuanTable(luan,new ArrayList(ic.getFileNames())); 320 LuanTable fileNames = new LuanTable(luan,new ArrayList(ic.getFileNames()));
322 return fn.call(luan,new Object[]{dir,fileNames}); 321 return fn.call(luan,new Object[]{dir,fileNames});
355 354
356 DocFn(IndexSearcher searcher) { 355 DocFn(IndexSearcher searcher) {
357 this.searcher = searcher; 356 this.searcher = searcher;
358 } 357 }
359 358
360 @Override public Object call(LuanState luan,Object[] args) throws LuanException { 359 @Override public Object call(Luan luan,Object[] args) throws LuanException {
361 try { 360 try {
362 return toTable(luan,searcher.doc(docID)); 361 return toTable(luan,searcher.doc(docID));
363 } catch(IOException e) { 362 } catch(IOException e) {
364 throw new LuanException(e); 363 throw new LuanException(e);
365 } 364 }
401 400
402 public void ensure_open() throws IOException { 401 public void ensure_open() throws IOException {
403 close(openSearcher()); 402 close(openSearcher());
404 } 403 }
405 404
406 public int advanced_search( final LuanState luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException { 405 public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException {
407 Utils.checkNotNull(queryStr); 406 Utils.checkNotNull(queryStr);
408 Query query = SaneQueryParser.parseQuery(mfp,queryStr); 407 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
409 IndexSearcher searcher = threadLocalSearcher.get(); 408 IndexSearcher searcher = threadLocalSearcher.get();
410 boolean inTransaction = searcher != null; 409 boolean inTransaction = searcher != null;
411 if( !inTransaction ) 410 if( !inTransaction )
450 if( !inTransaction ) 449 if( !inTransaction )
451 close(searcher); 450 close(searcher);
452 } 451 }
453 } 452 }
454 453
455 public Object search_in_transaction(LuanState luan,LuanFunction fn) throws LuanException, IOException { 454 public Object search_in_transaction(Luan luan,LuanFunction fn) throws LuanException, IOException {
456 if( threadLocalSearcher.get() != null ) 455 if( threadLocalSearcher.get() != null )
457 throw new LuanException("can't nest search_in_transaction calls"); 456 throw new LuanException("can't nest search_in_transaction calls");
458 IndexSearcher searcher = openSearcher(); 457 IndexSearcher searcher = openSearcher();
459 threadLocalSearcher.set(searcher); 458 threadLocalSearcher.set(searcher);
460 try { 459 try {
555 if( s != null ) 554 if( s != null )
556 return s; 555 return s;
557 throw new LuanException("invalid field type for "+ifld); 556 throw new LuanException("invalid field type for "+ifld);
558 } 557 }
559 558
560 private static LuanTable toTable(LuanState luan,Document doc) throws LuanException { 559 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
561 if( doc==null ) 560 if( doc==null )
562 return null; 561 return null;
563 LuanTable table = new LuanTable(luan); 562 LuanTable table = new LuanTable(luan);
564 for( IndexableField ifld : doc ) { 563 for( IndexableField ifld : doc ) {
565 String name = ifld.name(); 564 String name = ifld.name();
587 public String highlightTerm(String originalText,TokenGroup tokenGroup) { 586 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
588 return originalText; 587 return originalText;
589 } 588 }
590 }; 589 };
591 590
592 public LuanFunction highlighter(LuanState luan,String queryStr,LuanFunction formatter,final Integer fragmentSize,String dotdotdot) throws ParseException { 591 public LuanFunction highlighter(Luan luan,String queryStr,LuanFunction formatter,final Integer fragmentSize,String dotdotdot) throws ParseException {
593 Query query = SaneQueryParser.parseQuery(mfp,queryStr); 592 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
594 Formatter fmt = new Formatter() { 593 Formatter fmt = new Formatter() {
595 public String highlightTerm(String originalText,TokenGroup tokenGroup) { 594 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
596 if( tokenGroup.getTotalScore() <= 0 ) 595 if( tokenGroup.getTotalScore() <= 0 )
597 return originalText; 596 return originalText;
607 if( chooser != null ) 606 if( chooser != null )
608 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); 607 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
609 final Highlighter hl = new Highlighter(fmt,queryScorer); 608 final Highlighter hl = new Highlighter(fmt,queryScorer);
610 hl.setTextFragmenter( new NullFragmenter() ); 609 hl.setTextFragmenter( new NullFragmenter() );
611 return new LuanFunction() { 610 return new LuanFunction() {
612 @Override public String call(LuanState luan,Object[] args) throws LuanException { 611 @Override public String call(Luan luan,Object[] args) throws LuanException {
613 String text = (String)args[0]; 612 String text = (String)args[0];
614 try { 613 try {
615 if( chooser != null ) { 614 if( chooser != null ) {
616 String s = chooser.getBestFragment(analyzer,null,text); 615 String s = chooser.getBestFragment(analyzer,null,text);
617 if( s != null ) { 616 if( s != null ) {