Mercurial Hosting > luan

changeset 1345:6f8988830098

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
unique LuceneIndex per dir
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 25 Feb 2019 11:00:10 -0700
parents dc2af9d5463b
children efd1c6380f2c
files src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java
diffstat 2 files changed, 87 insertions(+), 43 deletions(-) [+]
line wrap: on
line diff
diff -r dc2af9d5463b -r 6f8988830098 src/luan/modules/lucene/Lucene.luan
--- a/src/luan/modules/lucene/Lucene.luan	Mon Feb 25 07:00:55 2019 -0700
+++ b/src/luan/modules/lucene/Lucene.luan	Mon Feb 25 11:00:10 2019 -0700
@@ -12,10 +12,7 @@
 local Rpc = require "luan:Rpc.luan"
 local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex"
 local NumberFieldParser = require "java:luan.lib.queryparser.NumberFieldParser"
-local StringFieldParser = require "java:luan.lib.queryparser.StringFieldParser"
 local SaneQueryParser = require "java:luan.lib.queryparser.SaneQueryParser"
-local Version = require "java:org.apache.lucene.util.Version"
-local EnglishAnalyzer = require "java:org.apache.lucene.analysis.en.EnglishAnalyzer"
 
 
 local Lucene = {}
@@ -23,12 +20,11 @@
 Lucene.instances = {}
 
 Lucene.type = {
-	string = LuceneIndex.STRING_FIELD_PARSER;
-	integer = NumberFieldParser.INT;
-	long = NumberFieldParser.LONG;
-	double = NumberFieldParser.DOUBLE;
-
-	english = StringFieldParser.new(EnglishAnalyzer.new(Version.LUCENE_CURRENT))
+	english = LuceneIndex.ENGLISH_FIELD_PARSER
+	string = LuceneIndex.STRING_FIELD_PARSER
+	integer = NumberFieldParser.INT
+	long = NumberFieldParser.LONG
+	double = NumberFieldParser.DOUBLE
 }
 
 Lucene.literal = SaneQueryParser.literal
@@ -36,9 +32,9 @@
 function Lucene.index(index_dir,default_type,default_fields)
 	local index = {}
 	index.dir = index_dir
-	local java_index = LuceneIndex.new(index_dir,default_type,default_fields)
+	local java_index, closer = LuceneIndex.getLuceneIndex(index_dir,default_type,default_fields)
 	index.java = java_index
---	index.indexed_fields = java_index.indexedFieldsMeta.newTable()
+	index.closer = closer or error()
 
 	index.indexed_fields = {}
 	local mt = {}
@@ -70,7 +66,7 @@
 
 	function index.close()
 		Lucene.instances[index] = nil
-		java_index.close()
+		closer.close()
 	end
 
 	function index.search( query, from, to, options )
@@ -148,7 +144,7 @@
 			local lucene_dir = uri("file:"..index.dir)
 			local before_restore = lucene_dir.parent().child("before_restore.zip")
 			index.zip(before_restore)
-			java_index.close()
+			java_index.doClose()
 			lucene_dir.delete()
 			Io.uri("os:unzip "..zip_file.canonical().to_string(),{dir=lucene_dir.parent()}).read_text()
 			java_index.reopen()
diff -r dc2af9d5463b -r 6f8988830098 src/luan/modules/lucene/LuceneIndex.java
--- a/src/luan/modules/lucene/LuceneIndex.java	Mon Feb 25 07:00:55 2019 -0700
+++ b/src/luan/modules/lucene/LuceneIndex.java	Mon Feb 25 11:00:10 2019 -0700
@@ -5,15 +5,15 @@
 import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Set;
 import java.util.HashSet;
 import java.util.Collections;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StoredField;
@@ -78,13 +79,63 @@
 import luan.lib.logging.LoggerFactory;
 
 
-public final class LuceneIndex implements Closeable {
+public final class LuceneIndex {
 	private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
 
+	private static final class Closer implements Closeable {
+		final LuceneIndex li;
+		boolean isClosed = false;
+		private final Exception created = new Exception("created");
+
+		Closer(Luan luan,LuceneIndex li) {
+			this.li = li;
+			luan.onClose(this);
+		}
+
+		public void close() throws IOException {
+			if( !isClosed ) {
+				li.close();
+				isClosed = true;
+			}
+		}
+
+		protected void finalize() throws Throwable {
+			if( !isClosed ) {
+				logger.error("not closed",created);
+				close();
+			}
+			super.finalize();
+		}
+	}
+
+	private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
+
+	public static Object[] getLuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
+		throws LuanException, IOException
+	{
+		String key = new File(indexDirStr).getCanonicalPath();
+		synchronized(indexes) {
+			LuceneIndex li = indexes.get(key);
+			if( li == null ) {
+				li = new LuceneIndex(indexDirStr,defaultFieldParser,defaultFields,key);
+				li.openCount = 1;
+				indexes.put(key,li);
+			} else {
+				if( defaultFieldParser != li.defaultFieldParser )
+					throw new LuanException("default_type doesn't match previous use");
+				if( !Arrays.equals(defaultFields,li.defaultFields) )
+					throw new LuanException("default_fields don't match previous use");
+				li.openCount++;
+			}
+			return new Object[]{li,new Closer(luan,li)};
+		}
+	}
+
+	private static final Version version = Version.LUCENE_4_9;
 	private static final String FLD_NEXT_ID = "nextId";
 	public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
+	public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
 
-	private static final Version version = Version.LUCENE_4_9;
 	private final ReentrantLock writeLock = new ReentrantLock();
 	private final File indexDir;
 	private SnapshotDeletionPolicy snapshotDeletionPolicy;
@@ -92,20 +143,26 @@
 	private DirectoryReader reader;
 	private IndexSearcher searcher;
 	private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
-	private boolean isClosed = true;
 	private final MultiFieldParser mfp;
 	private final Analyzer analyzer;
-	private final Exception created = new Exception("created");
 
-	private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
 	private File fileDir;
 	private int writeCount;
+	private AtomicInteger writeCounter = new AtomicInteger();
 
 	private Set<String> indexOnly = new HashSet<String>();
 
-	public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
+	private int openCount;
+	private final String key;
+	private final FieldParser defaultFieldParser;
+	private final String[] defaultFields;
+
+	private LuceneIndex(String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields,String key)
 		throws LuanException, IOException
 	{
+		this.key = key;
+		this.defaultFieldParser = defaultFieldParser;
+		this.defaultFields = defaultFields;
 		mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
 		mfp.fields.put( "type", STRING_FIELD_PARSER );
 		mfp.fields.put( "id", NumberFieldParser.LONG );
@@ -117,19 +174,15 @@
 			analyzer = sfp.analyzer;
 		}
 		this.analyzer = analyzer;
-		luan.onClose(this);
 		reopen();
 	}
 
 	public void reopen() throws LuanException, IOException {
-		if( !isClosed )  throw new RuntimeException();
-		isClosed = false;
 		IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
 		snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
 		conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
 		FSDirectory dir = FSDirectory.open(indexDir);
 		fileDir = dir.getDirectory();
-		globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger());
 		writer = new IndexWriter(dir,conf);
 		writer.commit();  // commit index creation
 		reader = DirectoryReader.open(dir);
@@ -137,12 +190,8 @@
 		initId();
 	}
 
-	private int globalWriteCount() {
-		return globalWriteCounters.get(fileDir).get();
-	}
-
 	private void wrote() {
-		globalWriteCounters.get(fileDir).incrementAndGet();
+		writeCounter.incrementAndGet();
 	}
 
 	public void delete_all() throws IOException {
@@ -312,24 +361,23 @@
 		return writer.getDirectory().toString();
 	}
 
-	public void close() throws IOException {
-		if( !isClosed ) {
-			writer.close();
-			reader.close();
-			isClosed = true;
+	private synchronized void close() throws IOException {
+		if( openCount > 0 ) {
+			if( --openCount == 0 ) {
+				doClose();
+				synchronized(indexes) {
+					indexes.remove(key);
+				}
+			}
 		}
 	}
 
-	protected void finalize() throws Throwable {
-		if( !isClosed ) {
-			logger.error("not closed",created);
-			close();
-		}
-		super.finalize();
+	public void doClose() throws IOException {
+		writer.close();
+		reader.close();
 	}
 
 
-
 	private static class DocFn extends LuanFunction {
 		final IndexSearcher searcher;
 		final Query query;
@@ -370,7 +418,7 @@
 	}
 
 	private synchronized IndexSearcher openSearcher() throws IOException {
-		int gwc = globalWriteCount();
+		int gwc = writeCounter.get();
 		if( writeCount != gwc ) {
 			writeCount = gwc;
 			DirectoryReader newReader = DirectoryReader.openIfChanged(reader);