repo/luan: src/luan/modules/lucene/LuceneIndex.java comparison

comparison src/luan/modules/lucene/LuceneIndex.java @ 1341:a015a0b5c388

add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()

author	Franklin Schmidt <fschmidt@gmail.com>
date	Tue, 19 Feb 2019 08:14:40 -0700
parents	8b61c8c4e07a
children	60599adc27b8

comparison

equal deleted inserted replaced

-:b3c4fcf29a53
+:a015a0b5c388
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.zip.ZipOutputStream;
 import java.util.zip.ZipEntry;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 		BytesRef br = new BytesRef();
 		NumericUtils.longToPrefixCoded(value,0,br);
 		return new Term(key,br);
 	}
-	public void delete(Luan luan,String queryStr) throws LuanException, IOException, ParseException {
+	public void delete(Luan luan,String queryStr)
+		throws LuanException, IOException, ParseException
+	{
 		Query query = SaneQueryParser.parseQuery(mfp,queryStr);
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
 		try {
 		indexedOnlyFields.putIfAbsent(type,new ConcurrentHashMap<String,LuanFunction>());
 		Map<String,LuanFunction> map = indexedOnlyFields.get(type);
 		map.put(field,fn);
 	}
-	public void save(Luan luan,LuanTable doc) throws LuanException, IOException {
+	public void save(Luan luan,LuanTable doc,LuanTable boosts)
+		throws LuanException, IOException
+	{
 		Set indexedOnlySet = new HashSet();
 		Object typeObj = doc.get("type");
 		if( typeObj==null )
 			throw new LuanException("missing 'type' field");
 		if( !(typeObj instanceof String) )
 		writeLock.lock();
 		try {
 			if( id == null ) {
 				id = nextId(luan);
 				doc.put("id",id);
-				writer.addDocument(toLucene(doc,indexedOnlySet));
+				writer.addDocument(toLucene(doc,indexedOnlySet,boosts));
 			} else {
-				writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet) );
+				writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet,boosts) );
 			}
 			if(commit) writer.commit();
 		} finally {
 			wrote();
 			writeLock.unlock();
 		if( ++id > idLim ) {
 			idLim += idBatch;
 			LuanTable doc = new LuanTable(luan);
 			doc.rawPut( "type", "next_id" );
 			doc.rawPut( FLD_NEXT_ID, idLim );
-			writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET));
+			writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET,null));
 			wrote();
 		}
 		return id;
 	}
 	public void ensure_open() throws IOException {
 		close(openSearcher());
 	}
-	public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException {
+	public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr )
+		throws LuanException, IOException, ParseException
+	{
 		Utils.checkNotNull(queryStr);
 		Query query = SaneQueryParser.parseQuery(mfp,queryStr);
 		IndexSearcher searcher = threadLocalSearcher.get();
 		boolean inTransaction = searcher != null;
 		if( !inTransaction )
 		}
 		mfp.fields.put( field, fp );
 	}
-	private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed)
+	private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed,Float boost)
+		throws LuanException
+	{
+		IndexableField fld = newField2(name,value,store,indexed);
+		if( boost != null )
+			((Field)fld).setBoost(boost);
+		return fld;
+	}
+	private IndexableField newField2(String name,Object value,Field.Store store,Set<String> indexed)
 		throws LuanException
 	{
 		if( value instanceof String ) {
 			String s = (String)value;
 			FieldParser fp = mfp.fields.get(name);
 			return new StoredField(name, b);
 		} else
 			throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
 	}
-	private Document toLucene(LuanTable table,Set indexOnly) throws LuanException {
+	private Document toLucene(LuanTable table,Set indexOnly,LuanTable boosts) throws LuanException {
 		Set<String> indexed = mfp.fields.keySet();
 		Document doc = new Document();
 		for( Map.Entry<Object,Object> entry : table.iterable() ) {
 			Object key = entry.getKey();
 			if( !(key instanceof String) )
 				throw new LuanException("key must be string");
 			String name = (String)key;
 			Object value = entry.getValue();
 			Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
+			Float boost = null;
+			if( boosts != null ) {
+				Object obj = boosts.get(name);
+				if( obj != null ) {
+					if( !(obj instanceof Number) )
+						throw new LuanException("boost '"+name+"' must be number");
+					boost = ((Number)obj).floatValue();
+				}
+			}
 			if( !(value instanceof LuanTable) ) {
-				doc.add(newField(name, value, store, indexed));
+				doc.add(newField( name, value, store, indexed, boost ));
 			} else { // list
 				LuanTable list = (LuanTable)value;
 				for( Object el : list.asList() ) {
-					doc.add(newField(name, el, store, indexed));
+					doc.add(newField( name, el, store, indexed, boost ));
 				}
 			}
 		}
 		return doc;
 	}
 					throw new RuntimeException(e);
 				}
 			}
 		};
 	}
+	public int count_tokens(String text)
+		throws IOException
+	{
+		int n = 0;
+		TokenStream ts = analyzer.tokenStream(null,text);
+		ts.reset();
+		while( ts.incrementToken() ) {
+			n++;
+		}
+		ts.close();
+		return n;
+	}
 }

Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1341:a015a0b5c388