Mercurial Hosting > luan
changeset 1345:6f8988830098
unique LuceneIndex per dir
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Mon, 25 Feb 2019 11:00:10 -0700 |
parents | dc2af9d5463b |
children | efd1c6380f2c |
files | src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java |
diffstat | 2 files changed, 87 insertions(+), 43 deletions(-) [+] |
line wrap: on
line diff
--- a/src/luan/modules/lucene/Lucene.luan Mon Feb 25 07:00:55 2019 -0700 +++ b/src/luan/modules/lucene/Lucene.luan Mon Feb 25 11:00:10 2019 -0700 @@ -12,10 +12,7 @@ local Rpc = require "luan:Rpc.luan" local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex" local NumberFieldParser = require "java:luan.lib.queryparser.NumberFieldParser" -local StringFieldParser = require "java:luan.lib.queryparser.StringFieldParser" local SaneQueryParser = require "java:luan.lib.queryparser.SaneQueryParser" -local Version = require "java:org.apache.lucene.util.Version" -local EnglishAnalyzer = require "java:org.apache.lucene.analysis.en.EnglishAnalyzer" local Lucene = {} @@ -23,12 +20,11 @@ Lucene.instances = {} Lucene.type = { - string = LuceneIndex.STRING_FIELD_PARSER; - integer = NumberFieldParser.INT; - long = NumberFieldParser.LONG; - double = NumberFieldParser.DOUBLE; - - english = StringFieldParser.new(EnglishAnalyzer.new(Version.LUCENE_CURRENT)) + english = LuceneIndex.ENGLISH_FIELD_PARSER + string = LuceneIndex.STRING_FIELD_PARSER + integer = NumberFieldParser.INT + long = NumberFieldParser.LONG + double = NumberFieldParser.DOUBLE } Lucene.literal = SaneQueryParser.literal @@ -36,9 +32,9 @@ function Lucene.index(index_dir,default_type,default_fields) local index = {} index.dir = index_dir - local java_index = LuceneIndex.new(index_dir,default_type,default_fields) + local java_index, closer = LuceneIndex.getLuceneIndex(index_dir,default_type,default_fields) index.java = java_index --- index.indexed_fields = java_index.indexedFieldsMeta.newTable() + index.closer = closer or error() index.indexed_fields = {} local mt = {} @@ -70,7 +66,7 @@ function index.close() Lucene.instances[index] = nil - java_index.close() + closer.close() end function index.search( query, from, to, options ) @@ -148,7 +144,7 @@ local lucene_dir = uri("file:"..index.dir) local before_restore = lucene_dir.parent().child("before_restore.zip") index.zip(before_restore) - java_index.close() + java_index.doClose() lucene_dir.delete() Io.uri("os:unzip "..zip_file.canonical().to_string(),{dir=lucene_dir.parent()}).read_text() java_index.reopen()
--- a/src/luan/modules/lucene/LuceneIndex.java Mon Feb 25 07:00:55 2019 -0700 +++ b/src/luan/modules/lucene/LuceneIndex.java Mon Feb 25 11:00:10 2019 -0700 @@ -5,15 +5,15 @@ import java.io.FileOutputStream; import java.io.FileInputStream; import java.io.IOException; +import java.util.Arrays; import java.util.Iterator; import java.util.Map; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Set; import java.util.HashSet; import java.util.Collections; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StoredField; @@ -78,13 +79,63 @@ import luan.lib.logging.LoggerFactory; -public final class LuceneIndex implements Closeable { +public final class LuceneIndex { private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); + private static final class Closer implements Closeable { + final LuceneIndex li; + boolean isClosed = false; + private final Exception created = new Exception("created"); + + Closer(Luan luan,LuceneIndex li) { + this.li = li; + luan.onClose(this); + } + + public void close() throws IOException { + if( !isClosed ) { + li.close(); + isClosed = true; + } + } + + protected void finalize() throws Throwable { + if( !isClosed ) { + logger.error("not closed",created); + close(); + } + super.finalize(); + } + } + + private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>(); + + public static Object[] getLuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) + throws LuanException, IOException + { + String key = new File(indexDirStr).getCanonicalPath(); + synchronized(indexes) { + LuceneIndex li = indexes.get(key); + if( li == null ) { + li = new LuceneIndex(indexDirStr,defaultFieldParser,defaultFields,key); + li.openCount = 1; + indexes.put(key,li); + } else { + if( defaultFieldParser != li.defaultFieldParser ) + throw new LuanException("default_type doesn't match previous use"); + if( !Arrays.equals(defaultFields,li.defaultFields) ) + throw new LuanException("default_fields don't match previous use"); + li.openCount++; + } + return new Object[]{li,new Closer(luan,li)}; + } + } + + private static final Version version = Version.LUCENE_4_9; private static final String FLD_NEXT_ID = "nextId"; public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); + public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version)); - private static final Version version = Version.LUCENE_4_9; private final ReentrantLock writeLock = new ReentrantLock(); private final File indexDir; private SnapshotDeletionPolicy snapshotDeletionPolicy; @@ -92,20 +143,26 @@ private DirectoryReader reader; private IndexSearcher searcher; private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); - private boolean isClosed = true; private final MultiFieldParser mfp; private final Analyzer analyzer; - private final Exception created = new Exception("created"); - private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); private File fileDir; private int writeCount; + private AtomicInteger writeCounter = new AtomicInteger(); private Set<String> indexOnly = new HashSet<String>(); - public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) + private int openCount; + private final String key; + private final FieldParser defaultFieldParser; + private final String[] defaultFields; + + private LuceneIndex(String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields,String key) throws LuanException, IOException { + this.key = key; + this.defaultFieldParser = defaultFieldParser; + this.defaultFields = defaultFields; mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); mfp.fields.put( "type", STRING_FIELD_PARSER ); mfp.fields.put( "id", NumberFieldParser.LONG ); @@ -117,19 +174,15 @@ analyzer = sfp.analyzer; } this.analyzer = analyzer; - luan.onClose(this); reopen(); } public void reopen() throws LuanException, IOException { - if( !isClosed ) throw new RuntimeException(); - isClosed = false; IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); conf.setIndexDeletionPolicy(snapshotDeletionPolicy); FSDirectory dir = FSDirectory.open(indexDir); fileDir = dir.getDirectory(); - globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger()); writer = new IndexWriter(dir,conf); writer.commit(); // commit index creation reader = DirectoryReader.open(dir); @@ -137,12 +190,8 @@ initId(); } - private int globalWriteCount() { - return globalWriteCounters.get(fileDir).get(); - } - private void wrote() { - globalWriteCounters.get(fileDir).incrementAndGet(); + writeCounter.incrementAndGet(); } public void delete_all() throws IOException { @@ -312,24 +361,23 @@ return writer.getDirectory().toString(); } - public void close() throws IOException { - if( !isClosed ) { - writer.close(); - reader.close(); - isClosed = true; + private synchronized void close() throws IOException { + if( openCount > 0 ) { + if( --openCount == 0 ) { + doClose(); + synchronized(indexes) { + indexes.remove(key); + } + } } } - protected void finalize() throws Throwable { - if( !isClosed ) { - logger.error("not closed",created); - close(); - } - super.finalize(); + public void doClose() throws IOException { + writer.close(); + reader.close(); } - private static class DocFn extends LuanFunction { final IndexSearcher searcher; final Query query; @@ -370,7 +418,7 @@ } private synchronized IndexSearcher openSearcher() throws IOException { - int gwc = globalWriteCount(); + int gwc = writeCounter.get(); if( writeCount != gwc ) { writeCount = gwc; DirectoryReader newReader = DirectoryReader.openIfChanged(reader);