Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1343:7d9a1f8894b0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene change indexed_only_field() to indexed_only_fields()
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 22 Feb 2019 10:12:05 -0700
parents 60599adc27b8
children dc2af9d5463b
comparison
equal deleted inserted replaced
1342:60599adc27b8 1343:7d9a1f8894b0
70 import luan.lib.parser.ParseException; 70 import luan.lib.parser.ParseException;
71 import luan.modules.Utils; 71 import luan.modules.Utils;
72 import luan.Luan; 72 import luan.Luan;
73 import luan.LuanTable; 73 import luan.LuanTable;
74 import luan.LuanFunction; 74 import luan.LuanFunction;
75 import luan.LuanCloner;
76 import luan.LuanException; 75 import luan.LuanException;
77 import luan.LuanRuntimeException; 76 import luan.LuanRuntimeException;
78 import luan.lib.logging.Logger; 77 import luan.lib.logging.Logger;
79 import luan.lib.logging.LoggerFactory; 78 import luan.lib.logging.LoggerFactory;
80 79
100 99
101 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); 100 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
102 private File fileDir; 101 private File fileDir;
103 private int writeCount; 102 private int writeCount;
104 103
105 private final ConcurrentMap<String,Map<String,LuanFunction>> indexedOnlyFields = new ConcurrentHashMap<String,Map<String,LuanFunction>>(); 104 private Set<String> indexOnly = new HashSet<String>();
106 private final Luan luanMine = new Luan();
107 private final LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE);
108 105
109 public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) 106 public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
110 throws LuanException, IOException 107 throws LuanException, IOException
111 { 108 {
112 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 109 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
181 wrote(); 178 wrote();
182 writeLock.unlock(); 179 writeLock.unlock();
183 } 180 }
184 } 181 }
185 182
186 public void indexed_only_field(String type,String field,LuanFunction fn) { 183 public void indexed_only_fields(List<String> fields) {
187 fn = (LuanFunction)cloner.get(fn); 184 indexOnly.addAll(fields);
188 indexedOnlyFields.putIfAbsent(type,new ConcurrentHashMap<String,LuanFunction>());
189 Map<String,LuanFunction> map = indexedOnlyFields.get(type);
190 map.put(field,fn);
191 } 185 }
192 186
193 public void save(Luan luan,LuanTable doc,LuanTable boosts) 187 public void save(Luan luan,LuanTable doc,LuanTable boosts)
194 throws LuanException, IOException 188 throws LuanException, IOException
195 { 189 {
196 Set indexedOnlySet = new HashSet();
197 Object typeObj = doc.get("type");
198 if( typeObj==null )
199 throw new LuanException("missing 'type' field");
200 if( !(typeObj instanceof String) )
201 throw new LuanException("type must be string");
202 String type = (String)typeObj;
203 Map<String,LuanFunction> map = indexedOnlyFields.get(type);
204 if( map != null ) {
205 for( Map.Entry<String,LuanFunction> entry : map.entrySet() ) {
206 String name = entry.getKey();
207 LuanFunction fn = entry.getValue();
208 Object value = Luan.first(fn.call(doc));
209 doc.put( name, value );
210 indexedOnlySet.add(name);
211 }
212 }
213 Object obj = doc.get("id"); 190 Object obj = doc.get("id");
214 Long id; 191 Long id;
215 try { 192 try {
216 id = (Long)obj; 193 id = (Long)obj;
217 } catch(ClassCastException e) { 194 } catch(ClassCastException e) {
222 writeLock.lock(); 199 writeLock.lock();
223 try { 200 try {
224 if( id == null ) { 201 if( id == null ) {
225 id = nextId(luan); 202 id = nextId(luan);
226 doc.put("id",id); 203 doc.put("id",id);
227 writer.addDocument(toLucene(doc,indexedOnlySet,boosts)); 204 writer.addDocument(toLucene(doc,boosts));
228 } else { 205 } else {
229 writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet,boosts) ); 206 writer.updateDocument( term("id",id), toLucene(doc,boosts) );
230 } 207 }
231 if(commit) writer.commit(); 208 if(commit) writer.commit();
232 } finally { 209 } finally {
233 wrote(); 210 wrote();
234 writeLock.unlock(); 211 writeLock.unlock();
286 if( ++id > idLim ) { 263 if( ++id > idLim ) {
287 idLim += idBatch; 264 idLim += idBatch;
288 LuanTable doc = new LuanTable(luan); 265 LuanTable doc = new LuanTable(luan);
289 doc.rawPut( "type", "next_id" ); 266 doc.rawPut( "type", "next_id" );
290 doc.rawPut( FLD_NEXT_ID, idLim ); 267 doc.rawPut( FLD_NEXT_ID, idLim );
291 writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET,null)); 268 writer.updateDocument(new Term("type","next_id"),toLucene(doc,null));
292 wrote(); 269 wrote();
293 } 270 }
294 return id; 271 return id;
295 } 272 }
296 273
493 } 470 }
494 mfp.fields.put( field, fp ); 471 mfp.fields.put( field, fp );
495 } 472 }
496 473
497 474
498 private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed,Float boost) 475 private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
499 throws LuanException 476 throws LuanException
500 { 477 {
501 IndexableField fld = newField2(name,value,store,indexed); 478 boolean hasBoost = boost!=null;
502 if( boost != null ) 479 IndexableField fld = newField2(name,value,indexed,hasBoost);
480 if( hasBoost )
503 ((Field)fld).setBoost(boost); 481 ((Field)fld).setBoost(boost);
504 return fld; 482 return fld;
505 } 483 }
506 484
507 private IndexableField newField2(String name,Object value,Field.Store store,Set<String> indexed) 485 private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
508 throws LuanException 486 throws LuanException
509 { 487 {
488 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
510 if( value instanceof String ) { 489 if( value instanceof String ) {
511 String s = (String)value; 490 String s = (String)value;
512 FieldParser fp = mfp.fields.get(name); 491 FieldParser fp = mfp.fields.get(name);
513 if( fp != null ) { 492 if( fp != null ) {
514 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) { 493 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
515 return new TextField(name, s, store); 494 return new TextField(name, s, store);
495 } else if (hasBoost) {
496 // fuck you modern lucene developers
497 return new Field(name, s, store, Field.Index.NOT_ANALYZED);
516 } else { 498 } else {
517 return new StringField(name, s, store); 499 return new StringField(name, s, store);
518 } 500 }
519 } else { 501 } else {
520 return new StoredField(name, s); 502 return new StoredField(name, s);
545 return new StoredField(name, b); 527 return new StoredField(name, b);
546 } else 528 } else
547 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'"); 529 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
548 } 530 }
549 531
550 private Document toLucene(LuanTable table,Set indexOnly,LuanTable boosts) throws LuanException { 532 private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
551 Set<String> indexed = mfp.fields.keySet(); 533 Set<String> indexed = mfp.fields.keySet();
552 Document doc = new Document(); 534 Document doc = new Document();
553 for( Map.Entry<Object,Object> entry : table.iterable() ) { 535 for( Map.Entry<Object,Object> entry : table.iterable() ) {
554 Object key = entry.getKey(); 536 Object key = entry.getKey();
555 if( !(key instanceof String) ) 537 if( !(key instanceof String) )
556 throw new LuanException("key must be string"); 538 throw new LuanException("key must be string");
557 String name = (String)key; 539 String name = (String)key;
558 Object value = entry.getValue(); 540 Object value = entry.getValue();
559 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
560 Float boost = null; 541 Float boost = null;
561 if( boosts != null ) { 542 if( boosts != null ) {
562 Object obj = boosts.get(name); 543 Object obj = boosts.get(name);
563 if( obj != null ) { 544 if( obj != null ) {
564 if( !(obj instanceof Number) ) 545 if( !(obj instanceof Number) )
565 throw new LuanException("boost '"+name+"' must be number"); 546 throw new LuanException("boost '"+name+"' must be number");
566 boost = ((Number)obj).floatValue(); 547 boost = ((Number)obj).floatValue();
567 } 548 }
568 } 549 }
569 if( !(value instanceof LuanTable) ) { 550 if( !(value instanceof LuanTable) ) {
570 doc.add(newField( name, value, store, indexed, boost )); 551 doc.add(newField( name, value, indexed, boost ));
571 } else { // list 552 } else { // list
572 LuanTable list = (LuanTable)value; 553 LuanTable list = (LuanTable)value;
573 for( Object el : list.asList() ) { 554 for( Object el : list.asList() ) {
574 doc.add(newField( name, el, store, indexed, boost )); 555 doc.add(newField( name, el, indexed, boost ));
575 } 556 }
576 } 557 }
577 } 558 }
578 return doc; 559 return doc;
579 } 560 }