Mercurial Hosting > luan

changeset 544:c5a93767cc5c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene overhaul, untested
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 12 Jun 2015 19:11:44 -0600
parents 9767da72545b
children ddcd4296107a
files lucene/ext/lucene-queryparser-4.9.0.jar lucene/ext/sane-lucene-queryparser.jar lucene/src/luan/modules/lucene/Ab_testing.luan lucene/src/luan/modules/lucene/FieldTable.java lucene/src/luan/modules/lucene/LuanQueryNodeProcessor.java lucene/src/luan/modules/lucene/Lucene.luan lucene/src/luan/modules/lucene/LuceneDocument.java lucene/src/luan/modules/lucene/LuceneIndex.java lucene/src/luan/modules/lucene/LuceneSearcher.java lucene/src/luan/modules/lucene/LuceneWriter.java lucene/src/luan/modules/lucene/Web_search.luan scripts/test.luan
diffstat 12 files changed, 134 insertions(+), 356 deletions(-) [+]
line wrap: on
line diff
diff -r 9767da72545b -r c5a93767cc5c lucene/ext/lucene-queryparser-4.9.0.jar
Binary file lucene/ext/lucene-queryparser-4.9.0.jar has changed
diff -r 9767da72545b -r c5a93767cc5c lucene/ext/sane-lucene-queryparser.jar
Binary file lucene/ext/sane-lucene-queryparser.jar has changed
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/Ab_testing.luan
--- a/lucene/src/luan/modules/lucene/Ab_testing.luan	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/Ab_testing.luan	Fri Jun 12 19:11:44 2015 -0600
@@ -10,6 +10,7 @@
 local Io = require "luan:Io"
 local Http = require "luan:http/Http"
 local Logging = require "luan:logging/Logging"
+local Lucene = require "luan:lucene/Lucene"
 
 local M = {}
 
@@ -34,8 +35,8 @@
 		-- test.date_field is optional
 
 		local field = "ab_test_" .. test.name
-		index.fields[field] == nil or error("test "+test.name+" already defined")
-		index.fields[field] = field .. " index"
+		index.indexed_fields[field] == nil or error("test "+test.name+" already defined")
+		index.indexed_fields[field] = Lucene.type.string
 		test.field = field
 
 		-- returns map of event name to (map of value to result) and "start_date"
@@ -52,7 +53,7 @@
 					for name, factory in pairs(test.aggregator_factories) do
 						aggregators[name] = factory()
 					end
-					local query = index.query.term{ [field] = value }
+					local query = field..":"..value
 					searcher.search(query, function(doc)
 						for _, aggregator in pairs(aggregators) do
 							aggregator.aggregate(doc)
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/FieldTable.java
--- a/lucene/src/luan/modules/lucene/FieldTable.java	Mon Jun 08 01:11:08 2015 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,64 +0,0 @@
-package luan.modules.lucene;
-
-import java.util.Map;
-import java.util.HashMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.Iterator;
-import luan.LuanState;
-import luan.LuanTable;
-import luan.LuanMeta;
-
-
-class FieldTable extends LuanMeta {
-	final Map<String,String> map = new ConcurrentHashMap<String,String>();
-	final Map<String,String> reverseMap = new ConcurrentHashMap<String,String>();
-
-	FieldTable() {
-		put("type","type index");
-		put("id","id index");
-	}
-
-	@Override public boolean canNewindex() {
-		return true;
-	}
-
-	@Override public void __new_index(LuanState luan,LuanTable tbl,Object key,Object value) {
-		put(key,value);
-	}
-
-	private void put(Object key,Object value) {
-		if( !(key instanceof String) )
-			throw new UnsupportedOperationException("key must be string");
-		String name = (String)key;
-		if( value==null ) {  // delete
-			reverseMap.remove(map.remove(name));
-			return;
-		}
-		if( !(value instanceof String) )
-			throw new UnsupportedOperationException("value must be string");
-		String field = (String)value;
-		String oldField = map.put(name,field);
-		if( oldField != null )
-			reverseMap.remove(oldField);
-		String oldName = reverseMap.put(field,name);
-		if( oldName != null ) {
-			reverseMap.put(field,oldName);
-			map.remove(name);
-			throw new IllegalArgumentException("field '"+oldName+"' is already assigned to '"+field+"'");
-		}
-	}
-
-	@Override public Object __index(LuanState luan,LuanTable tbl,Object key) {
-//	@Override public final Object get(Object key) {
-		return map.get(key);
-	}
-
-	@Override public final Iterator keys(LuanTable tbl) {
-		return map.keySet().iterator();
-	}
-
-	@Override protected String type(LuanTable tbl) {
-		return "lucene-field";
-	}
-
-}
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/LuanQueryNodeProcessor.java
--- a/lucene/src/luan/modules/lucene/LuanQueryNodeProcessor.java	Mon Jun 08 01:11:08 2015 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,52 +0,0 @@
-package luan.modules.lucene;
-
-import java.util.List;
-import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessor;
-import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler;
-import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
-import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode;
-import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
-
-
-public class LuanQueryNodeProcessor implements QueryNodeProcessor {
-	private final LuceneIndex index;
-	private final QueryNodeProcessor qnp;
-
-	public LuanQueryNodeProcessor(LuceneIndex index,QueryNodeProcessor qnp) {
-		this.index = index;
-		this.qnp = qnp;
-	}
-
-	public QueryConfigHandler getQueryConfigHandler() {
-		return qnp.getQueryConfigHandler();
-	}
-
-	public void setQueryConfigHandler(QueryConfigHandler queryConfigHandler) {
-		qnp.setQueryConfigHandler(queryConfigHandler);
-	}
-
-	public QueryNode process(QueryNode queryTree) throws QueryNodeException {
-		fix(queryTree);
-//		System.out.println(queryTree);
-		return qnp.process(queryTree);
-	}
-
-	private void fix(QueryNode queryTree) {
-		if( queryTree instanceof FieldQueryNode ) {
-			FieldQueryNode fqn = (FieldQueryNode)queryTree;
-			CharSequence fldSeq = fqn.getField();
-			if( fldSeq == null )
-				throw new RuntimeException("missing field for value: "+fqn.getText());
-			String fld = fldSeq.toString();
-			fld = index.map_field_name(fld);
-//			System.out.println("field = "+fld);
-			fqn.setField(fld);
-		}
-		List<QueryNode> list = queryTree.getChildren();
-		if( list != null ) {
-			for( QueryNode qn : list ) {
-				fix(qn);
-			}
-		}
-	}
-}
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/Lucene.luan
--- a/lucene/src/luan/modules/lucene/Lucene.luan	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/Lucene.luan	Fri Jun 12 19:11:44 2015 -0600
@@ -1,39 +1,28 @@
 java()
 local Luan = require "luan:Luan"
-local pairs = Luan.pairs
-local ipairs = Luan.ipairs
-local type = Luan.type
-local number_type = Luan.number_type
 local error = Luan.error
-local Number = require "luan:Number"
-local number_type = Number.number_type
-local Table = require "luan:Table"
-local unpack = Table.unpack
 local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex"
-local Term = require "java:org.apache.lucene.index.Term"
-local TermQuery = require "java:org.apache.lucene.search.TermQuery"
-local TermRangeQuery = require "java:org.apache.lucene.search.TermRangeQuery"
-local NumericRangeQuery = require "java:org.apache.lucene.search.NumericRangeQuery"
-local MatchAllDocsQuery = require "java:org.apache.lucene.search.MatchAllDocsQuery"
-local BooleanQuery = require "java:org.apache.lucene.search.BooleanQuery"
-local BooleanClause = require "java:org.apache.lucene.search.BooleanClause"
-local BytesRef = require "java:org.apache.lucene.util.BytesRef"
-local NumericUtils = require "java:org.apache.lucene.util.NumericUtils"
-local Sort = require "java:org.apache.lucene.search.Sort"
-local SortField = require "java:org.apache.lucene.search.SortField"
+local NumberFieldParser = require "java:sane.lucene.queryparser.NumberFieldParser"
+
 
 local M = {}
 
+M.type = {
+	string = LuceneIndex.STRING_FIELD_PARSER;
+	integer = NumberFieldParser.INT;
+	long = NumberFieldParser.LONG;
+	double = NumberFieldParser.DOUBLE;
+}
+
 function M.index(indexDir)
 	local index = {}
 	local java_index = LuceneIndex.new(indexDir)
-	index.fields = java_index.fields.newTable()
+	index.indexed_fields = java_index.indexedFieldsMeta.newTable()
 	index.to_string = java_index.to_string
 	index.backup = java_index.backup
 	index.Writer = java_index.Writer
 	index.Searcher = java_index.Searcher
 	index.delete_all = java_index.delete_all
-	index.map_field_name = java_index.map_field_name
 	index.close = java_index.close
 
 	function index.save_document(doc)
@@ -70,120 +59,6 @@
 		end )
 	end
 
-
-
-	local queryTbl = {}
-	index.query = queryTbl
-
-	queryTbl.parse = java_index.parse
-
-	queryTbl.all_docs = MatchAllDocsQuery.new()
-	
-	function queryTbl.term(field,value)
-		if value==nil and type(field)=="table" then
-			local iter = pairs(field)
-			field, value = iter()
-			iter()==nil or error "only one term allowed"
-		end
-		field and value or error "missing term"
-		field = index.map_field_name(field)
-		local tp = type(value)
-		if tp == "string" then
-			return TermQuery.new(Term.new(field,value))
-		end
-		if tp == "number" then
-			local br = BytesRef.new()
-			tp = number_type(value)
-			if tp == "integer" then
-				NumericUtils.intToPrefixCoded(value,0,br)
-			elseif tp == "long" then
-				NumericUtils.longToPrefixCoded(value,0,br)
-			elseif tp == "double" then
-				value = NumericUtils.doubleToSortableLong(value)
-				NumericUtils.longToPrefixCoded(value,0,br)
-			else
-				error("invalid term value type: "..tp)
-			end
-			return TermQuery.new(Term.new(field,br))
-		end
-		error("invalid term value type: "..tp)
-	end
-
-	function queryTbl.range(field,min,max,include_min,include_max)
-		if include_min == nil then include_min = true end
-		if include_max == nil then include_max = true end
-		field = index.map_field_name(field)
-		local tp = type(min)
-		tp == type(max) or error "min and max types don't match"
-		if tp == "string" then
-			return TermRangeQuery.newStringRange(field,min,max,include_min,include_max)
-		end
-		if tp == "number" then
-			tp = number_type(min)
-			tp == number_type(max) or error "min and max number types don't match"
-			if tp == "integer" then
-				return NumericRangeQuery.newIntRange(field,min,max,include_min,include_max)
-			end
-			if tp == "long" then
-				return NumericRangeQuery.newLongRange(field,min,max,include_min,include_max)
-			end
-			if tp == "double" then
-				return NumericRangeQuery.newDoubleRange(field,min,max,include_min,include_max)
-			end
-			error("invalid term value type: "..tp)
-		end
-		error("invalid term value type: "..tp)
-	end
-
-	function queryTbl.boolean(t)
-		local boolean_query = BooleanQuery.new()
-		for query, occur_string in pairs(t) do
-			local occur = BooleanClause.Occur.valueOf( occur_string.upper() )
-			boolean_query.add( query, occur )
-		end
-		return boolean_query
-	end
-	
-	-- and list
-	function queryTbl.all(t)
-		local bt = {}
-		for key, query in pairs(t) do
-			if type(key)=="string" then
-				query = queryTbl.term{[key]=query}
-			end
-			bt[query] = "MUST"
-		end
-		return queryTbl.boolean(bt)
-	end
-	
-	-- or list
-	function queryTbl.any(t)
-		local bt = {}
-		for key, query in pairs(t) do
-			if type(key)=="string" then
-				query = queryTbl.term{[key]=query}
-			end
-			bt[query] = "SHOULD"
-		end
-		return queryTbl.boolean(bt)
-	end
-	
-	
-	function queryTbl.sort(fields)
-		#fields > 0 or error "list of sort fields expected"
-		local a = {}
-		for _, f in ipairs(fields) do
-			f.field or error "missing sort field"
-			f.type or error "missing sort type"
-			local field = index.map_field_name(f.field)
-			local type = SortField.Type.valueOf( f.type.upper() )
-			local reverse = f.reverse or false
-			a[#a+1] = SortField.new(field,type,reverse)
-		end
-		return Sort.new(unpack(a))
-	end
-
-
 	return index
 end
 
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/LuceneDocument.java
--- a/lucene/src/luan/modules/lucene/LuceneDocument.java	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/LuceneDocument.java	Fri Jun 12 19:11:44 2015 -0600
@@ -21,12 +21,9 @@
 
 
 public class LuceneDocument {
-	// I assume there will be more flags later
-	public static final String INDEX = "index";
-
 	private LuceneDocument(String a) {}  // never
 
-	static Document toLucene(LuanState luan,LuanTable table,Map<String,String> nameMap) throws LuanException {
+	static Document toLucene(LuanState luan,LuanTable table,Set<String> indexed) throws LuanException {
 		Document doc = new Document();
 		for( Map.Entry<Object,Object> entry : table.iterable(luan) ) {
 			Object key = entry.getKey();
@@ -34,38 +31,30 @@
 				throw luan.exception("key must be string");
 			String name = (String)key;
 			Object value = entry.getValue();
-			String newName = nameMap.get(name);
-			if( newName != null )
-				name = newName;
-			Set<String> flags = new HashSet<String>();
-			String[] a = name.split(" +");
-			for( int i=1; i<a.length; i++ ) {
-				flags.add(a[i]);
-			}
 			if( value instanceof String ) {
 				String s = (String)value;
-				if( flags.remove(INDEX) ) {
+				if( indexed.contains(name) ) {
 					doc.add(new StringField(name, s, Field.Store.YES));
 				} else {
 					doc.add(new StoredField(name, s));
 				}
 			} else if( value instanceof Integer ) {
 				int i = (Integer)value;
-				if( flags.remove(INDEX) ) {
+				if( indexed.contains(name) ) {
 					doc.add(new IntField(name, i, Field.Store.YES));
 				} else {
 					doc.add(new StoredField(name, i));
 				}
 			} else if( value instanceof Long ) {
 				long i = (Long)value;
-				if( flags.remove(INDEX) ) {
+				if( indexed.contains(name) ) {
 					doc.add(new LongField(name, i, Field.Store.YES));
 				} else {
 					doc.add(new StoredField(name, i));
 				}
 			} else if( value instanceof Double ) {
 				double i = (Double)value;
-				if( flags.remove(INDEX) ) {
+				if( indexed.contains(name) ) {
 					doc.add(new DoubleField(name, i, Field.Store.YES));
 				} else {
 					doc.add(new StoredField(name, i));
@@ -75,21 +64,16 @@
 				doc.add(new StoredField(name, b));
 			} else
 				throw luan.exception("invalid value type "+value.getClass()+"' for '"+name+"'");
-			if( !flags.isEmpty() )
-				throw luan.exception("invalid flags "+flags+" in '"+name+"'");
 		}
 		return doc;
 	}
 
-	static LuanTable toTable(LuanState luan,Document doc,Map<String,String> nameMap) throws LuanException {
+	static LuanTable toTable(LuanState luan,Document doc) throws LuanException {
 		if( doc==null )
 			return null;
 		LuanTable table = new LuanTable();
 		for( IndexableField ifld : doc ) {
 			String name = ifld.name();
-			String newName = nameMap.get(name);
-			if( newName != null )
-				name = newName;
 			BytesRef br = ifld.binaryValue();
 			if( br != null ) {
 				table.rawPut(name,br.bytes);
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/LuceneIndex.java
--- a/lucene/src/luan/modules/lucene/LuceneIndex.java	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/LuceneIndex.java	Fri Jun 12 19:11:44 2015 -0600
@@ -5,12 +5,13 @@
 import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.Iterator;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.zip.ZipOutputStream;
 import java.util.zip.ZipEntry;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -23,8 +24,15 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
-import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.IndexSearcher;
+import sane.lucene.queryparser.SaneQueryParser;
+import sane.lucene.queryparser.FieldParser;
+import sane.lucene.queryparser.MultiFieldParser;
+import sane.lucene.queryparser.StringFieldParser;
+import sane.lucene.queryparser.NumberFieldParser;
+import sane.lucene.queryparser.ParseException;
 import luan.modules.Utils;
 import luan.Luan;
 import luan.LuanState;
@@ -32,6 +40,7 @@
 import luan.LuanFunction;
 import luan.LuanJavaFunction;
 import luan.LuanException;
+import luan.LuanMeta;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,6 +49,8 @@
 	private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
 
 	private static final String FLD_NEXT_ID = "nextId";
+	private static final Analyzer analyzer = new KeywordAnalyzer();
+	public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer);
 
 	final Lock writeLock = new ReentrantLock();
 	private final File indexDir;
@@ -47,15 +58,16 @@
 	final IndexWriter writer;
 	private DirectoryReader reader;
 	private LuceneSearcher searcher;
-	public final FieldTable fields = new FieldTable();
 	private boolean isClosed = false;
+	private final MultiFieldParser mfp = new MultiFieldParser();
 
 	public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException {
+		mfp.fields.put( "type", STRING_FIELD_PARSER );
+		mfp.fields.put( "id", NumberFieldParser.LONG );
 		File indexDir = new File(indexDirStr);
 		this.indexDir = indexDir;
 		Directory dir = FSDirectory.open(indexDir);
 		Version version = Version.LUCENE_4_9;
-		Analyzer analyzer = new StandardAnalyzer(version);
 		IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
 		snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
 		conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
@@ -68,20 +80,7 @@
 	}
 
 	Document toLucene(LuanState luan,LuanTable table) throws LuanException {
-		return LuceneDocument.toLucene(luan,table,fields.map);
-	}
-
-	LuanTable toTable(LuanState luan,Document doc) throws LuanException {
-		return LuceneDocument.toTable(luan,doc,fields.reverseMap);
-	}
-
-	public String map_field_name(String fld) {
-		String s = fields.map.get(fld);
-		return s!=null ? s : fld;
-	}
-
-	Term newTerm(String fld,String text) {
-		return new Term(map_field_name(fld),text);
+		return LuceneDocument.toLucene(luan,table,mfp.fields.keySet());
 	}
 
 	public LuceneWriter openWriter() {
@@ -120,13 +119,13 @@
 	private final int idBatch = 10;
 
 	private void initId(LuanState luan) throws LuanException, IOException {
-		TopDocs td = searcher.search(new TermQuery(newTerm("type","next_id")),1);
+		IndexSearcher searcher = this.searcher.searcher;
+		TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
 		switch(td.totalHits) {
 		case 0:
 			break;  // do nothing
 		case 1:
-			LuanTable doc = searcher.doc(luan,td.scoreDocs[0].doc);
-			idLim = (Long)doc.rawGet(FLD_NEXT_ID);
+			idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
 			id = idLim;
 			break;
 		default:
@@ -134,16 +133,15 @@
 		}
 	}
 
-	synchronized String nextId(LuanState luan) throws LuanException, IOException {
-		String rtn = Long.toString(++id);
-		if( id > idLim ) {
+	synchronized long nextId(LuanState luan) throws LuanException, IOException {
+		if( ++id > idLim ) {
 			idLim += idBatch;
 			LuanTable doc = new LuanTable();
 			doc.rawPut( "type", "next_id" );
 			doc.rawPut( FLD_NEXT_ID, idLim );
-			writer.updateDocument(newTerm("type","next_id"),toLucene(luan,doc));
+			writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc));
 		}
-		return rtn;
+		return id;
 	}
 
 
@@ -210,10 +208,47 @@
 	}
 
 
-	public Query parse(String s) throws QueryNodeException {
-		StandardQueryParser qp = new StandardQueryParser();
-		qp.setQueryNodeProcessor(new LuanQueryNodeProcessor(this,qp.getQueryNodeProcessor()));
-		return qp.parse(s,null);
+
+	public final LuanMeta indexedFieldsMeta = new LuanMeta() {
+
+		@Override public boolean canNewindex() {
+			return true;
+		}
+
+		@Override public Object __index(LuanState luan,LuanTable tbl,Object key) {
+			return mfp.fields.get(key);
+		}
+
+		@Override public void __new_index(LuanState luan,LuanTable tbl,Object key,Object value) throws LuanException {
+			if( !(key instanceof String) )
+				throw luan.exception("key must be string");
+			String field = (String)key;
+			if( value==null ) {  // delete
+				mfp.fields.remove(field);
+				return;
+			}
+			if( !(value instanceof FieldParser) )
+				throw luan.exception("value must be FieldParser like the values of Lucene.type");
+			FieldParser parser = (FieldParser)value;
+			mfp.fields.put( field, parser );
+		}
+
+		@Override public final Iterator keys(LuanTable tbl) {
+			return mfp.fields.keySet().iterator();
+		}
+
+		@Override protected String type(LuanTable tbl) {
+			return "lucene-indexed-fields";
+		}
+
+	};
+
+	public Query parseQuery(String s) throws ParseException {
+		return SaneQueryParser.parseQuery(mfp,s);
+	}
+
+	public Sort parseSort(String s) throws ParseException {
+		return SaneQueryParser.parseSort(mfp,s);
 	}
 
 }
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/LuceneSearcher.java
--- a/lucene/src/luan/modules/lucene/LuceneSearcher.java	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/LuceneSearcher.java	Fri Jun 12 19:11:44 2015 -0600
@@ -21,7 +21,7 @@
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
+import sane.lucene.queryparser.ParseException;
 import luan.Luan;
 import luan.LuanState;
 import luan.LuanTable;
@@ -30,11 +30,12 @@
 import luan.LuanException;
 import luan.LuanRuntimeException;
 import luan.LuanMethod;
+import luan.modules.Utils;
 
 
 public final class LuceneSearcher {
 	private final LuceneIndex index;
-	private final IndexSearcher searcher;
+	final IndexSearcher searcher;
 
 	LuceneSearcher(LuceneIndex index,IndexReader reader) {
 		this.index = index;
@@ -46,10 +47,10 @@
 		searcher.getIndexReader().decRef();
 	}
 
-	LuanTable doc(LuanState luan,int docID) throws LuanException, IOException {
-		return index.toTable(luan,searcher.doc(docID));
+	private LuanTable doc(LuanState luan,int docID) throws LuanException, IOException {
+		return LuceneDocument.toTable(luan,searcher.doc(docID));
 	}
-
+/*
 	TopDocs search(Query query,int n) throws IOException {
 		return searcher.search(query,n);
 	}
@@ -57,7 +58,7 @@
 	TopFieldDocs search(Query query,int n,Sort sort) throws IOException {
 		return searcher.search(query,n,sort);
 	}
-
+*/
 	// luan
 
 	private static final LuanFunction nothingFn = new LuanFunction() {
@@ -78,15 +79,9 @@
 		}
 	}
 
-	@LuanMethod public Object[] search( final LuanState luan, Object queryObj, Object nObj, Sort sort ) throws LuanException, IOException, QueryNodeException {
-		Query query;
-		if( queryObj instanceof Query ) {
-			query = (Query)queryObj;
-		} else if( queryObj instanceof String ) {
-			String s = (String)queryObj;
-			query = index.parse(s);
-		} else
-			throw luan.exception("bad argument #1 (string or Query expected, got "+Luan.type(queryObj)+")");
+	@LuanMethod public Object[] search( final LuanState luan, String queryStr, Object nObj, String sortStr ) throws LuanException, IOException, ParseException {
+		Utils.checkNotNull(luan,queryStr);
+		Query query = index.parseQuery(queryStr);
 		if( nObj instanceof LuanFunction ) {
 			final LuanFunction fn = (LuanFunction)nObj;
 			Collector col = new MyCollector() {
@@ -119,6 +114,7 @@
 			searcher.search(query,thcc);
 			return new Object[]{ nothingFn, 0, thcc.getTotalHits() };
 		}
+		Sort sort = sortStr==null ? null : index.parseSort(sortStr);
 		TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
 		final ScoreDoc[] scoreDocs = td.scoreDocs;
 		LuanFunction results = new LuanFunction() {
@@ -145,7 +141,7 @@
 	LuanTable table() {
 		LuanTable tbl = new LuanTable();
 		try {
-			add( tbl, "search", LuanState.class, Object.class, Object.class, Sort.class );
+			add( tbl, "search", LuanState.class, String.class, Object.class, String.class );
 		} catch(NoSuchMethodException e) {
 			throw new RuntimeException(e);
 		}
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/LuceneWriter.java
--- a/lucene/src/luan/modules/lucene/LuceneWriter.java	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/LuceneWriter.java	Fri Jun 12 19:11:44 2015 -0600
@@ -7,6 +7,8 @@
 import java.util.ArrayList;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.NumericUtils;
 import luan.Luan;
 import luan.LuanState;
 import luan.LuanTable;
@@ -31,12 +33,30 @@
 		index.writer.commit();
 	}
 
-	void addDocument(LuanState luan,LuanTable doc) throws LuanException, IOException {
-		index.writer.addDocument(index.toLucene(luan,doc));
+	private Term term(String key,int value) {
+		BytesRef br = new BytesRef();
+		NumericUtils.intToPrefixCoded(value,0,br);
+		return new Term(key,br);
+	}
+
+	private Term term(String key,long value) {
+		BytesRef br = new BytesRef();
+		NumericUtils.longToPrefixCoded(value,0,br);
+		return new Term(key,br);
 	}
 
-	void updateDocument(LuanState luan,Term term,LuanTable doc) throws LuanException, IOException {
-		index.writer.updateDocument(term,index.toLucene(luan,doc));
+	private Term term(LuanState luan,String key,Object value) throws LuanException {
+		if( value instanceof String )
+			return new Term( key, (String)value );
+		if( value instanceof Integer )
+			return term( key, (Integer)value );
+		if( value instanceof Long )
+			return term( key, (Long)value );
+		if( value instanceof Float )
+			return term( key, NumericUtils.floatToSortableInt((Float)value) );
+		if( value instanceof Double )
+			return term( key, NumericUtils.doubleToSortableLong((Double)value) );
+		throw luan.exception("invalid value type '"+value.getClass().getSimpleName()+"' for key '"+key+"'");
 	}
 
 	public void delete_documents(LuanState luan,LuanTable tblTerms) throws LuanException, IOException {
@@ -46,27 +66,21 @@
 			Object value = entry.getValue();
 			if( !(key instanceof String) )
 				throw luan.exception("key must be a string but got "+key.getClass().getSimpleName());
-			if( !(value instanceof String) )
-				throw luan.exception("value must be a string but got "+value.getClass().getSimpleName());
-			list.add( index.newTerm( (String)key, (String)value ) );
+			list.add( term( luan, (String)key, value ) );
 		}
 		index.writer.deleteDocuments(list.toArray(new Term[list.size()]));
 	}
 
-	String nextId(LuanState luan) throws LuanException, IOException {
-		return index.nextId(luan);
-	}
-
 	public void save_document(LuanState luan,LuanTable doc) throws LuanException, IOException {
 		if( doc.get(luan,"type")==null )
 			throw luan.exception("missing 'type' field");
-		String id = (String)doc.get(luan,"id");
+		Long id = (Long)doc.get(luan,"id");
 		if( id == null ) {
-			id = nextId(luan);
+			id = index.nextId(luan);
 			doc.put(luan,"id",id);
-			addDocument(luan,doc);
+			index.writer.addDocument(index.toLucene(luan,doc));
 		} else {
-			updateDocument(luan,index.newTerm("id",id),doc);
+			index.writer.updateDocument( term("id",id), index.toLucene(luan,doc) );
 		}
 	}
 
diff -r 9767da72545b -r c5a93767cc5c lucene/src/luan/modules/lucene/Web_search.luan
--- a/lucene/src/luan/modules/lucene/Web_search.luan	Mon Jun 08 01:11:08 2015 -0400
+++ b/lucene/src/luan/modules/lucene/Web_search.luan	Fri Jun 12 19:11:44 2015 -0600
@@ -4,10 +4,6 @@
 local ipairs = Luan.ipairs
 local range = Luan.range
 local to_string = Luan.to_string
-local Number = require "luan:Number"
-local integer = Number.integer
-local long = Number.long
-local double = Number.double
 local Io = require "luan:Io"
 local Http = require "luan:http/Http"
 local String = require "luan:String"
@@ -31,8 +27,8 @@
 						<label>Query:</label>
 					</div>
 					<div colspan=10>
-						<input name="query" size="80" value="query.all_docs" autofocus />
-						<div textcolor="#888">Query examples: <i>query.term{ type = 'user' }</i> or <i>"type:user AND name:Joe"</i></div>
+						<input name="query" size="80" autofocus />
+						<div textcolor="#888">Query examples: <i>type:user</i> or <i>+type:user +name:Joe"</i></div>
 					</div>
 				</div>
 				<div row margin-top="1em">
@@ -49,7 +45,7 @@
 					</div>
 					<div colspan=10>
 						<input name="sort" size="60" />
-						<div textcolor="#888">Sort examples: sort{{ field = 'id', type='int' }}</div>
+						<div textcolor="#888">Sort examples: <i>name, id</i></div>
 					</div>
 				</div>
 				<div row margin-top="1em">
@@ -130,20 +126,13 @@
 
 	return function()
 		Io.stdout = Http.response.text_writer()
-		local query_string = Http.request.parameter.query
-		if query_string == nil then
+		local query = Http.request.parameter.query
+		if query == nil then
 			form()
 			return
 		end
-		local query_env = {
-			query=index.query;
-			integer=integer;
-			long=long;
-			double=double;
-		}
-		local query = load(query_string,"<query>",query_env,true)()
 		local rows = string_to_number(Http.request.parameter.rows)
-		local sort = load(Http.request.parameter.sort,"<sort>",{sort=index.query.sort},true)()
+		local sort = Http.request.parameter.sort
 		index.Searcher( function(searcher)
 			local results, length, total_hits = searcher.search(query,rows,sort)
 			local headers = {}
diff -r 9767da72545b -r c5a93767cc5c scripts/test.luan
--- a/scripts/test.luan	Mon Jun 08 01:11:08 2015 -0400
+++ b/scripts/test.luan	Fri Jun 12 19:11:44 2015 -0600
@@ -83,7 +83,7 @@
 web_search()
 
 init()
-Http.request.parameter.query = "query.all_docs"
+Http.request.parameter.query = ""
 Http.request.parameter.rows = "100"
 Http.request.parameter.sort = ""
 web_search()