Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1267:9fa8b8389578

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add LuanTable.luan; support metatable __gc(); add luan.sql;
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 12 Nov 2018 02:10:41 -0700
parents 475905984870
children 48f302bdc187
comparison
equal deleted inserted replaced
1266:05934fbf635a 1267:9fa8b8389578
92 private DirectoryReader reader; 92 private DirectoryReader reader;
93 private IndexSearcher searcher; 93 private IndexSearcher searcher;
94 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); 94 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
95 private boolean isClosed = true; 95 private boolean isClosed = true;
96 private final MultiFieldParser mfp; 96 private final MultiFieldParser mfp;
97 public final LuanTable indexed_only_fields = new LuanTable(); 97 public final LuanTable indexed_only_fields;
98 private final Analyzer analyzer; 98 private final Analyzer analyzer;
99 private final Exception created = new Exception("created"); 99 private final Exception created = new Exception("created");
100 100
101 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); 101 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
102 private File fileDir; 102 private File fileDir;
103 private int writeCount; 103 private int writeCount;
104 104
105 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) throws LuanException, IOException { 105 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
106 throws LuanException, IOException
107 {
108 indexed_only_fields = new LuanTable(luan); // not good, not thread safe
106 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 109 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
107 mfp.fields.put( "type", STRING_FIELD_PARSER ); 110 mfp.fields.put( "type", STRING_FIELD_PARSER );
108 mfp.fields.put( "id", NumberFieldParser.LONG ); 111 mfp.fields.put( "id", NumberFieldParser.LONG );
109 File indexDir = new File(indexDirStr); 112 File indexDir = new File(indexDirStr);
110 this.indexDir = indexDir; 113 this.indexDir = indexDir;
175 } 178 }
176 } 179 }
177 180
178 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException { 181 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException {
179 Set indexedOnlySet = new HashSet(); 182 Set indexedOnlySet = new HashSet();
180 Object typeObj = doc.get(luan,"type"); 183 Object typeObj = doc.get("type");
181 if( typeObj==null ) 184 if( typeObj==null )
182 throw new LuanException("missing 'type' field"); 185 throw new LuanException("missing 'type' field");
183 if( !(typeObj instanceof String) ) 186 if( !(typeObj instanceof String) )
184 throw new LuanException("type must be string"); 187 throw new LuanException("type must be string");
185 String type = (String)typeObj; 188 String type = (String)typeObj;
186 Object indexedOnlyObj = indexed_only_fields.get(luan,type); 189 Object indexedOnlyObj = indexed_only_fields.get(type);
187 if( indexedOnlyObj != null ) { 190 if( indexedOnlyObj != null ) {
188 if( !(indexedOnlyObj instanceof LuanTable) ) 191 if( !(indexedOnlyObj instanceof LuanTable) )
189 throw new LuanException("indexed_only_fields elements must be tables"); 192 throw new LuanException("indexed_only_fields elements must be tables");
190 LuanTable indexedOnly = (LuanTable)indexedOnlyObj; 193 LuanTable indexedOnly = (LuanTable)indexedOnlyObj;
191 for( Map.Entry<Object,Object> entry : indexedOnly.iterable(luan) ) { 194 for( Map.Entry<Object,Object> entry : indexedOnly.iterable() ) {
192 Object key = entry.getKey(); 195 Object key = entry.getKey();
193 if( !(key instanceof String) ) 196 if( !(key instanceof String) )
194 throw new LuanException("indexed_only_fields."+type+" entries must be strings"); 197 throw new LuanException("indexed_only_fields."+type+" entries must be strings");
195 String name = (String)key; 198 String name = (String)key;
196 Object value = entry.getValue(); 199 Object value = entry.getValue();
197 if( !(value instanceof LuanFunction) ) 200 if( !(value instanceof LuanFunction) )
198 throw new LuanException("indexed_only_fields."+type+" values must be functions"); 201 throw new LuanException("indexed_only_fields."+type+" values must be functions");
199 LuanFunction fn = (LuanFunction)value; 202 LuanFunction fn = (LuanFunction)value;
200 value = Luan.first(fn.call(luan,new Object[]{doc})); 203 value = Luan.first(fn.call(luan,new Object[]{doc}));
201 doc.put(luan, name, value ); 204 doc.put( name, value );
202 indexedOnlySet.add(name); 205 indexedOnlySet.add(name);
203 } 206 }
204 } 207 }
205 Object obj = doc.get(luan,"id"); 208 Object obj = doc.get("id");
206 Long id; 209 Long id;
207 try { 210 try {
208 id = (Long)obj; 211 id = (Long)obj;
209 } catch(ClassCastException e) { 212 } catch(ClassCastException e) {
210 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); 213 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
213 boolean commit = !writeLock.isHeldByCurrentThread(); 216 boolean commit = !writeLock.isHeldByCurrentThread();
214 writeLock.lock(); 217 writeLock.lock();
215 try { 218 try {
216 if( id == null ) { 219 if( id == null ) {
217 id = nextId(luan); 220 id = nextId(luan);
218 doc.put(luan,"id",id); 221 doc.put("id",id);
219 writer.addDocument(toLucene(luan,doc,indexedOnlySet)); 222 writer.addDocument(toLucene(doc,indexedOnlySet));
220 } else { 223 } else {
221 writer.updateDocument( term("id",id), toLucene(luan,doc,indexedOnlySet) ); 224 writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet) );
222 } 225 }
223 if(commit) writer.commit(); 226 if(commit) writer.commit();
224 } finally { 227 } finally {
225 wrote(); 228 wrote();
226 writeLock.unlock(); 229 writeLock.unlock();
275 } 278 }
276 279
277 public synchronized long nextId(LuanState luan) throws LuanException, IOException { 280 public synchronized long nextId(LuanState luan) throws LuanException, IOException {
278 if( ++id > idLim ) { 281 if( ++id > idLim ) {
279 idLim += idBatch; 282 idLim += idBatch;
280 LuanTable doc = new LuanTable(); 283 LuanTable doc = new LuanTable(luan);
281 doc.rawPut( "type", "next_id" ); 284 doc.rawPut( "type", "next_id" );
282 doc.rawPut( FLD_NEXT_ID, idLim ); 285 doc.rawPut( FLD_NEXT_ID, idLim );
283 writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc,Collections.EMPTY_SET)); 286 writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET));
284 wrote(); 287 wrote();
285 } 288 }
286 return id; 289 return id;
287 } 290 }
288 291
312 315
313 public Object snapshot(LuanState luan,LuanFunction fn) throws LuanException, IOException { 316 public Object snapshot(LuanState luan,LuanFunction fn) throws LuanException, IOException {
314 IndexCommit ic = snapshotDeletionPolicy.snapshot(); 317 IndexCommit ic = snapshotDeletionPolicy.snapshot();
315 try { 318 try {
316 String dir = fileDir.toString(); 319 String dir = fileDir.toString();
317 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames())); 320 LuanTable fileNames = new LuanTable(luan,new ArrayList(ic.getFileNames()));
318 return fn.call(luan,new Object[]{dir,fileNames}); 321 return fn.call(luan,new Object[]{dir,fileNames});
319 } finally { 322 } finally {
320 snapshotDeletionPolicy.release(ic); 323 snapshotDeletionPolicy.release(ic);
321 } 324 }
322 } 325 }
353 this.searcher = searcher; 356 this.searcher = searcher;
354 } 357 }
355 358
356 @Override public Object call(LuanState luan,Object[] args) throws LuanException { 359 @Override public Object call(LuanState luan,Object[] args) throws LuanException {
357 try { 360 try {
358 return toTable(searcher.doc(docID)); 361 return toTable(luan,searcher.doc(docID));
359 } catch(IOException e) { 362 } catch(IOException e) {
360 throw new LuanException(e); 363 throw new LuanException(e);
361 } 364 }
362 } 365 }
363 } 366 }
516 return new StoredField(name, b); 519 return new StoredField(name, b);
517 } else 520 } else
518 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'"); 521 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
519 } 522 }
520 523
521 private Document toLucene(LuanState luan,LuanTable table,Set indexOnly) throws LuanException { 524 private Document toLucene(LuanTable table,Set indexOnly) throws LuanException {
522 Set<String> indexed = mfp.fields.keySet(); 525 Set<String> indexed = mfp.fields.keySet();
523 Document doc = new Document(); 526 Document doc = new Document();
524 for( Map.Entry<Object,Object> entry : table.iterable(luan) ) { 527 for( Map.Entry<Object,Object> entry : table.iterable() ) {
525 Object key = entry.getKey(); 528 Object key = entry.getKey();
526 if( !(key instanceof String) ) 529 if( !(key instanceof String) )
527 throw new LuanException("key must be string"); 530 throw new LuanException("key must be string");
528 String name = (String)key; 531 String name = (String)key;
529 Object value = entry.getValue(); 532 Object value = entry.getValue();
551 if( s != null ) 554 if( s != null )
552 return s; 555 return s;
553 throw new LuanException("invalid field type for "+ifld); 556 throw new LuanException("invalid field type for "+ifld);
554 } 557 }
555 558
556 private static LuanTable toTable(Document doc) throws LuanException { 559 private static LuanTable toTable(LuanState luan,Document doc) throws LuanException {
557 if( doc==null ) 560 if( doc==null )
558 return null; 561 return null;
559 LuanTable table = new LuanTable(); 562 LuanTable table = new LuanTable(luan);
560 for( IndexableField ifld : doc ) { 563 for( IndexableField ifld : doc ) {
561 String name = ifld.name(); 564 String name = ifld.name();
562 Object value = getValue(ifld); 565 Object value = getValue(ifld);
563 Object old = table.rawGet(name); 566 Object old = table.rawGet(name);
564 if( old == null ) { 567 if( old == null ) {
566 } else { 569 } else {
567 LuanTable list; 570 LuanTable list;
568 if( old instanceof LuanTable ) { 571 if( old instanceof LuanTable ) {
569 list = (LuanTable)old; 572 list = (LuanTable)old;
570 } else { 573 } else {
571 list = new LuanTable(); 574 list = new LuanTable(luan);
572 list.rawPut(1,old); 575 list.rawPut(1,old);
573 table.rawPut(name,list); 576 table.rawPut(name,list);
574 } 577 }
575 list.rawPut(list.rawLength()+1,value); 578 list.rawPut(list.rawLength()+1,value);
576 } 579 }