Mercurial Hosting > luan

changeset 1343:7d9a1f8894b0

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene change indexed_only_field() to indexed_only_fields()
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 22 Feb 2019 10:12:05 -0700
parents 60599adc27b8
children dc2af9d5463b
files conv.txt src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/sql/Sql.luan
diffstat 4 files changed, 21 insertions(+), 39 deletions(-) [+]
line wrap: on
line diff
--- a/conv.txt	Wed Feb 20 12:09:51 2019 -0700
+++ b/conv.txt	Fri Feb 22 10:12:05 2019 -0700
@@ -1,3 +1,4 @@
+indexed_only_field
 lucene search sort
 call
 LuanState
--- a/src/luan/modules/lucene/Lucene.luan	Wed Feb 20 12:09:51 2019 -0700
+++ b/src/luan/modules/lucene/Lucene.luan	Fri Feb 22 10:12:05 2019 -0700
@@ -63,7 +63,7 @@
 	index.ensure_open = java_index.ensure_open
 	index.next_id = java_index.nextId
 	index.highlighter = java_index.highlighter
-	index.indexed_only_field = java_index.indexed_only_field
+	index.indexed_only_fields = java_index.indexed_only_fields
 	index.count_tokens = java_index.count_tokens
 
 	Lucene.instances[index] = true
--- a/src/luan/modules/lucene/LuceneIndex.java	Wed Feb 20 12:09:51 2019 -0700
+++ b/src/luan/modules/lucene/LuceneIndex.java	Fri Feb 22 10:12:05 2019 -0700
@@ -72,7 +72,6 @@
 import luan.Luan;
 import luan.LuanTable;
 import luan.LuanFunction;
-import luan.LuanCloner;
 import luan.LuanException;
 import luan.LuanRuntimeException;
 import luan.lib.logging.Logger;
@@ -102,9 +101,7 @@
 	private File fileDir;
 	private int writeCount;
 
-	private final ConcurrentMap<String,Map<String,LuanFunction>> indexedOnlyFields = new ConcurrentHashMap<String,Map<String,LuanFunction>>();
-	private final Luan luanMine = new Luan();
-	private final LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE);
+	private Set<String> indexOnly = new HashSet<String>();
 
 	public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
 		throws LuanException, IOException
@@ -183,33 +180,13 @@
 		}
 	}
 
-	public void indexed_only_field(String type,String field,LuanFunction fn) {
-		fn = (LuanFunction)cloner.get(fn);
-		indexedOnlyFields.putIfAbsent(type,new ConcurrentHashMap<String,LuanFunction>());
-		Map<String,LuanFunction> map = indexedOnlyFields.get(type);
-		map.put(field,fn);
+	public void indexed_only_fields(List<String> fields) {
+		indexOnly.addAll(fields);
 	}
 
 	public void save(Luan luan,LuanTable doc,LuanTable boosts)
 		throws LuanException, IOException
 	{
-		Set indexedOnlySet = new HashSet();
-		Object typeObj = doc.get("type");
-		if( typeObj==null )
-			throw new LuanException("missing 'type' field");
-		if( !(typeObj instanceof String) )
-			throw new LuanException("type must be string");
-		String type = (String)typeObj;
-		Map<String,LuanFunction> map = indexedOnlyFields.get(type);
-		if( map != null ) {
-			for( Map.Entry<String,LuanFunction> entry : map.entrySet() ) {
-				String name = entry.getKey();
-				LuanFunction fn = entry.getValue();
-				Object value = Luan.first(fn.call(doc));
-				doc.put( name, value );
-				indexedOnlySet.add(name);
-			}
-		}
 		Object obj = doc.get("id");
 		Long id;
 		try {
@@ -224,9 +201,9 @@
 			if( id == null ) {
 				id = nextId(luan);
 				doc.put("id",id);
-				writer.addDocument(toLucene(doc,indexedOnlySet,boosts));
+				writer.addDocument(toLucene(doc,boosts));
 			} else {
-				writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet,boosts) );
+				writer.updateDocument( term("id",id), toLucene(doc,boosts) );
 			}
 			if(commit) writer.commit();
 		} finally {
@@ -288,7 +265,7 @@
 			LuanTable doc = new LuanTable(luan);
 			doc.rawPut( "type", "next_id" );
 			doc.rawPut( FLD_NEXT_ID, idLim );
-			writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET,null));
+			writer.updateDocument(new Term("type","next_id"),toLucene(doc,null));
 			wrote();
 		}
 		return id;
@@ -495,24 +472,29 @@
 	}
 
 
-	private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed,Float boost)
+	private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
 		throws LuanException
 	{
-		IndexableField fld = newField2(name,value,store,indexed);
-		if( boost != null )
+		boolean hasBoost = boost!=null;
+		IndexableField fld = newField2(name,value,indexed,hasBoost);
+		if( hasBoost )
 			((Field)fld).setBoost(boost);
 		return fld;
 	}
 
-	private IndexableField newField2(String name,Object value,Field.Store store,Set<String> indexed)
+	private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
 		throws LuanException
 	{
+		Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
 		if( value instanceof String ) {
 			String s = (String)value;
 			FieldParser fp = mfp.fields.get(name);
 			if( fp != null ) {
 				if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
 					return new TextField(name, s, store);
+				} else if (hasBoost) {
+					// fuck you modern lucene developers
+					return new Field(name, s, store, Field.Index.NOT_ANALYZED);
 				} else {
 					return new StringField(name, s, store);
 				}
@@ -547,7 +529,7 @@
 			throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
 	}
 
-	private Document toLucene(LuanTable table,Set indexOnly,LuanTable boosts) throws LuanException {
+	private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
 		Set<String> indexed = mfp.fields.keySet();
 		Document doc = new Document();
 		for( Map.Entry<Object,Object> entry : table.iterable() ) {
@@ -556,7 +538,6 @@
 				throw new LuanException("key must be string");
 			String name = (String)key;
 			Object value = entry.getValue();
-			Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
 			Float boost = null;
 			if( boosts != null ) {
 				Object obj = boosts.get(name);
@@ -567,11 +548,11 @@
 				}
 			}
 			if( !(value instanceof LuanTable) ) {
-				doc.add(newField( name, value, store, indexed, boost ));
+				doc.add(newField( name, value, indexed, boost ));
 			} else { // list
 				LuanTable list = (LuanTable)value;
 				for( Object el : list.asList() ) {
-					doc.add(newField( name, el, store, indexed, boost ));
+					doc.add(newField( name, el, indexed, boost ));
 				}
 			}
 		}
--- a/src/luan/modules/sql/Sql.luan	Wed Feb 20 12:09:51 2019 -0700
+++ b/src/luan/modules/sql/Sql.luan	Fri Feb 22 10:12:05 2019 -0700
@@ -48,7 +48,7 @@
 				return nil
 			end
 			return result
-		end
+		end, rs
 	end
 
 	return database