Mercurial Hosting > luan

changeset 547:0be287ab0309

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add lucene/Versioning and simplify Lucene fn names
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 15 Jun 2015 01:22:58 -0600
parents eaef1005ab87
children f4dfe9c64c25
files lucene/ext/sane-lucene-queryparser.jar lucene/src/luan/modules/lucene/Lucene.luan lucene/src/luan/modules/lucene/LuceneIndex.java lucene/src/luan/modules/lucene/Versioning.luan scripts/test.luan
diffstat 5 files changed, 81 insertions(+), 35 deletions(-) [+]
line wrap: on
line diff
Binary file lucene/ext/sane-lucene-queryparser.jar has changed
--- a/lucene/src/luan/modules/lucene/Lucene.luan	Sun Jun 14 22:17:58 2015 -0600
+++ b/lucene/src/luan/modules/lucene/Lucene.luan	Mon Jun 15 01:22:58 2015 -0600
@@ -3,6 +3,7 @@
 local error = Luan.error
 local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex"
 local NumberFieldParser = require "java:sane.lucene.queryparser.NumberFieldParser"
+local SaneQueryParser = require "java:sane.lucene.queryparser.SaneQueryParser"
 
 
 local M = {}
@@ -14,6 +15,8 @@
 	double = NumberFieldParser.DOUBLE;
 }
 
+M.literal = SaneQueryParser.literal
+
 function M.index(indexDir)
 	local index = {}
 	local java_index = LuceneIndex.new(indexDir)
@@ -23,8 +26,8 @@
 	index.advanced_search = java_index.advanced_search
 	index.search_in_transaction = java_index.search_in_transaction
 	index.delete_all = java_index.delete_all
-	index.delete_documents = java_index.delete_documents
-	index.save_document = java_index.save_document
+	index.delete = java_index.delete
+	index.save = java_index.save
 	index.update_in_transaction = java_index.update_in_transaction
 	index.close = java_index.close
 
--- a/lucene/src/luan/modules/lucene/LuceneIndex.java	Sun Jun 14 22:17:58 2015 -0600
+++ b/lucene/src/luan/modules/lucene/LuceneIndex.java	Mon Jun 15 01:22:58 2015 -0600
@@ -114,53 +114,26 @@
 		}
 	}
 
-	private static Term term(String key,int value) {
-		BytesRef br = new BytesRef();
-		NumericUtils.intToPrefixCoded(value,0,br);
-		return new Term(key,br);
-	}
-
 	private static Term term(String key,long value) {
 		BytesRef br = new BytesRef();
 		NumericUtils.longToPrefixCoded(value,0,br);
 		return new Term(key,br);
 	}
 
-	private static Term term(LuanState luan,String key,Object value) throws LuanException {
-		if( value instanceof String )
-			return new Term( key, (String)value );
-		if( value instanceof Integer )
-			return term( key, (Integer)value );
-		if( value instanceof Long )
-			return term( key, (Long)value );
-		if( value instanceof Float )
-			return term( key, NumericUtils.floatToSortableInt((Float)value) );
-		if( value instanceof Double )
-			return term( key, NumericUtils.doubleToSortableLong((Double)value) );
-		throw luan.exception("invalid value type '"+value.getClass().getSimpleName()+"' for key '"+key+"'");
-	}
-
-	public void delete_documents(LuanState luan,LuanTable tblTerms) throws LuanException, IOException {
-		List<Term> list = new ArrayList<Term>();
-		for( Map.Entry<Object,Object> entry : tblTerms.iterable(luan) ) {
-			Object key = entry.getKey();
-			Object value = entry.getValue();
-			if( !(key instanceof String) )
-				throw luan.exception("key must be a string but got "+key.getClass().getSimpleName());
-			list.add( term( luan, (String)key, value ) );
-		}
+	public void delete(LuanState luan,String queryStr) throws LuanException, IOException, ParseException {
+		Query query = SaneQueryParser.parseQuery(mfp,queryStr);
 
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
 		try {
-			writer.deleteDocuments(list.toArray(new Term[list.size()]));
+			writer.deleteDocuments(query);
 			if(commit) writer.commit();
 		} finally {
 			writeLock.unlock();
 		}
 	}
 
-	public void save_document(LuanState luan,LuanTable doc) throws LuanException, IOException {
+	public void save(LuanState luan,LuanTable doc) throws LuanException, IOException {
 		if( doc.get(luan,"type")==null )
 			throw luan.exception("missing 'type' field");
 		Long id = (Long)doc.get(luan,"id");
@@ -199,6 +172,20 @@
 
 	private void initId(LuanState luan) throws LuanException, IOException {
 		TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
+
+		// tmp hack
+		if( td.totalHits == 0 ) {
+			td = searcher.search(new TermQuery(new Term("type index","next_id")),1);
+			if( td.totalHits == 1 ) {
+				long idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
+				LuanTable doc = new LuanTable();
+				doc.rawPut( "type", "next_id" );
+				doc.rawPut( FLD_NEXT_ID, idLim );
+				writer.addDocument(toLucene(luan,doc));
+				writer.commit();
+			}
+		}
+
 		switch(td.totalHits) {
 		case 0:
 			break;  // do nothing
@@ -327,7 +314,7 @@
 				MyCollector col = new MyCollector() {
 					@Override public void collect(int doc) {
 						try {
-							docFn.docID = doc;
+							docFn.docID = docBase + doc;
 							luan.call(fn,new Object[]{++i,docFn});
 						} catch(LuanException e) {
 							throw new LuanRuntimeException(e);
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/lucene/src/luan/modules/lucene/Versioning.luan	Mon Jun 15 01:22:58 2015 -0600
@@ -0,0 +1,56 @@
+local Luan = require "luan:Luan"
+local error = Luan.error
+local pairs = Luan.pairs or error()
+local Number = require "luan:Number"
+local integer = Number.integer or error()
+local long = Number.long or error()
+local String = require "luan:String"
+local matches = String.matches or error()
+local sub = String.sub or error()
+local string_to_number = String.string_to_number or error()
+local Table = require "luan:Table"
+local clone = Table.clone or error()
+local Lucene = require "luan:lucene/Lucene"
+require "luan:logging/init"
+local Logging = require "luan:logging/Logging"
+
+local logger = Logging.logger "lucene versioning"
+
+
+local M = {}
+
+function M.update(db,steps,version)
+	local doc = db.get_document"type:version" or { type="version", version=integer(0) }
+	while doc.version < version do
+		doc.version = integer(doc.version + 1)
+		logger.error("step "..doc.version)
+		db.update_in_transaction( function()
+			local step = steps[doc.version]
+			step and step(db)
+			db.save(doc)
+		end )
+	end
+end
+
+
+-- hack to deal with latest changes
+function M.a_big_step(db)
+	db.indexed_fields["id index"] = Lucene.type.string
+	db.advanced_search( Lucene.literal"id index" .. ":*", function(_,doc_fn)
+		local doc = doc_fn()
+		for field, value in pairs(clone(doc)) do
+			if matches(field,".* index") then
+				local new_field = sub(field,1,-7)
+				db.indexed_fields[new_field] or error("field '"..new_field.."' not indexed")
+				doc[new_field] = value
+				doc[field] = nil
+			end
+		end
+		doc.id = long(string_to_number(doc.id))
+		db.save(doc)
+	end )
+	db.indexed_fields["type index"] = Lucene.type.string
+	db.delete( Lucene.literal"type index" .. ":*" )
+end
+
+return M
--- a/scripts/test.luan	Sun Jun 14 22:17:58 2015 -0600
+++ b/scripts/test.luan	Mon Jun 15 01:22:58 2015 -0600
@@ -70,7 +70,7 @@
 for record in range(1,10) do
 	local doc = {type="test"}
 	ab_testing.to_doc(doc)
-	db.save_document(doc)
+	db.save(doc)
 end
 
 init()