Mercurial Hosting > luan

changeset 1397:0dc9837c16be

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix lucene issues
author Franklin Schmidt <fschmidt@gmail.com>
date Wed, 11 Sep 2019 01:31:21 -0600
parents a5f61890ad84
children 67c0e47b5be3
files examples/blog/src/lib/Db.luan src/luan/modules/http/LuanHandler.java src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java
diffstat 4 files changed, 76 insertions(+), 73 deletions(-) [+]
line wrap: on
line diff
diff -r a5f61890ad84 -r 0dc9837c16be examples/blog/src/lib/Db.luan
--- a/examples/blog/src/lib/Db.luan	Tue Sep 10 13:41:22 2019 -0600
+++ b/examples/blog/src/lib/Db.luan	Wed Sep 11 01:31:21 2019 -0600
@@ -34,6 +34,6 @@
 
 Db.db = Db.new("site:/private/local/lucene")
 
---Db.db.restore_from_postgres()
+Db.db.restore_from_postgres()
 Db.db.check()
 return Db
diff -r a5f61890ad84 -r 0dc9837c16be src/luan/modules/http/LuanHandler.java
--- a/src/luan/modules/http/LuanHandler.java	Tue Sep 10 13:41:22 2019 -0600
+++ b/src/luan/modules/http/LuanHandler.java	Wed Sep 11 01:31:21 2019 -0600
@@ -180,8 +180,9 @@
 			LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE);
 			luan = (Luan)cloner.clone(currentLuan);
 		}
+		luan.load(text,"<eval_in_root>",null).call();
+		currentLuan.onClose = null;
 		luan.onClose = this;
-		luan.load(text,"<eval_in_root>",null).call();
 		currentLuan = luan;
 	}
 
@@ -278,7 +279,9 @@
 		for( String mod : mods ) {
 			if( loaded.rawGet(mod) == null ) {
 				LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE);
+				currentLuan.onClose = null;
 				currentLuan = (Luan)cloner.clone(currentLuan);
+				currentLuan.onClose = this;
 				break;
 			}
 		}
diff -r a5f61890ad84 -r 0dc9837c16be src/luan/modules/lucene/Lucene.luan
--- a/src/luan/modules/lucene/Lucene.luan	Tue Sep 10 13:41:22 2019 -0600
+++ b/src/luan/modules/lucene/Lucene.luan	Wed Sep 11 01:31:21 2019 -0600
@@ -41,13 +41,10 @@
 	type(index_dir)=="table" or error "index_dir must be table"
 	index_dir.to_uri_string and matches(index_dir.to_uri_string(),"^file:") or error "must be file"
 	options = options or {}
-	options.postgres_spec==nil or type(options.postgres_spec)=="table" or error "postgres_spec must be table"
-	options.completer==nil or type(options.completer)=="function" or error "completer must be table"
 	local index = {}
 	index.dir = index_dir
-	local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,options.default_type,options.default_fields,options.completer,options.postgres_spec)
+	local java_index = LuceneIndex.getLuceneIndex(index_dir.java.file,options)
 	index.java = java_index
-	index.closer = closer or error()
 	index.completer = options.completer
 
 	index.indexed_fields = {}
@@ -73,13 +70,14 @@
 	index.highlighter = java_index.highlighter
 	index.indexed_only_fields = java_index.indexed_only_fields
 	index.count_tokens = java_index.count_tokens
-	index.close = closer.close
+	index.close = java_index.close
 
 	index.has_postgres_backup = java_index.hasPostgresBackup()
 	function index.rebuild_postgres_backup()
 		java_index.rebuild_postgres_backup(index.completer)
 	end
 	index.restore_from_postgres = java_index.restore_from_postgres
+	index.force_restore_from_postgres = java_index.force_restore_from_postgres
 	function index.check()
 		java_index.check(index.completer)
 	end
diff -r a5f61890ad84 -r 0dc9837c16be src/luan/modules/lucene/LuceneIndex.java
--- a/src/luan/modules/lucene/LuceneIndex.java	Tue Sep 10 13:41:22 2019 -0600
+++ b/src/luan/modules/lucene/LuceneIndex.java	Wed Sep 11 01:31:21 2019 -0600
@@ -5,6 +5,8 @@
 import java.io.FileOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.lang.ref.Reference;
+import java.lang.ref.WeakReference;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -83,59 +85,25 @@
 import luan.lib.logging.LoggerFactory;
 
 
-public final class LuceneIndex {
+public final class LuceneIndex implements Closeable {
 	private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
 
-	private static final class Closer implements Closeable {
-		final LuceneIndex li;
-		boolean isClosed = false;
-		private final Exception created = new Exception("created");
-
-		Closer(Luan luan,LuceneIndex li) {
-			this.li = li;
-			luan.onClose(this);
-		}
+	private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
 
-		public void close() throws IOException {
-			if( !isClosed ) {
-				try {
-					li.close();
-				} catch(SQLException e) {
-					throw new RuntimeException(e);
-				}
-				isClosed = true;
-			}
-		}
-
-		protected void finalize() throws Throwable {
-			if( !isClosed ) {
-				sysLogger.error("not closed",created);
-				close();
-			}
-			super.finalize();
-		}
-	}
-
-	private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
-
-	public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,LuanFunction completer,LuanTable postgresSpec)
+	public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
 		throws LuanException, IOException, ClassNotFoundException, SQLException
 	{
 		String key = indexDir.getCanonicalPath();
 		synchronized(indexes) {
-			LuceneIndex li = indexes.get(key);
-			if( li == null ) {
-				li = new LuceneIndex(luan,indexDir,defaultFieldParser,defaultFields,key,completer,postgresSpec);
-				li.openCount = 1;
-				indexes.put(key,li);
-			} else {
-				if( defaultFieldParser != li.defaultFieldParser )
-					throw new LuanException("default_type doesn't match previous use");
-				if( !Arrays.equals(defaultFields,li.defaultFields) )
-					throw new LuanException("default_fields don't match previous use");
-				li.openCount++;
+			Reference<LuceneIndex> ref = indexes.get(key);
+			if( ref != null ) {
+				LuceneIndex li = ref.get();
+				if( li != null )
+					li.closeWriter();
 			}
-			return new Object[]{li,new Closer(luan,li)};
+			LuceneIndex li = new LuceneIndex(luan,indexDir,options);
+			indexes.put(key, new WeakReference<LuceneIndex>(li));
+			return li;
 		}
 	}
 
@@ -162,18 +130,26 @@
 
 	private Set<String> indexOnly = new HashSet<String>();
 
-	private int openCount;
-	private final String key;
+	private boolean isClosed = false;
+	private final Exception created = new Exception("created");
 	private final FieldParser defaultFieldParser;
 	private final String[] defaultFields;
 
 	private final PostgresBackup postgresBackup;
+	private boolean wasCreated;
 
-	private LuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanFunction completer,LuanTable postgresSpec)
+	private LuceneIndex(Luan luan,File indexDir,LuanTable options)
 		throws LuanException, IOException, ClassNotFoundException, SQLException
 	{
+		Map map = options.asMap();
+		FieldParser defaultFieldParser = (FieldParser)map.remove("default_type");
+		LuanTable defaultFieldsTbl = Utils.removeTable(map,"default_fields");
+		String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
+		LuanFunction completer = Utils.removeFunction(map,"completer");
+		LuanTable postgresSpec = Utils.removeTable(map,"postgres_spec");
+		Utils.checkEmpty(map);
+
 		this.luanLogger = luan.getLogger(LuceneIndex.class);
-		this.key = key;
 		this.defaultFieldParser = defaultFieldParser;
 		this.defaultFields = defaultFields;
 		mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
@@ -186,7 +162,7 @@
 			analyzer = sfp.analyzer;
 		}
 		this.analyzer = analyzer;
-		boolean wasCreated = reopen();
+		wasCreated = reopen();
 		if( postgresSpec == null ) {
 			postgresBackup = null;
 		} else {
@@ -194,16 +170,25 @@
 				throw new LuanException("completer is required for postgres_spec");
 			Map spec = postgresSpec.asMap();
 			postgresBackup = new PostgresBackup(luan,spec);
-			if( postgresBackup != null ) {
-				if( !wasCreated && postgresBackup.wasCreated ) {
-					luanLogger.error("rebuilding postgres backup");
-					rebuild_postgres_backup(completer);
-				} else if( wasCreated && !postgresBackup.wasCreated ) {
-					luanLogger.error("restoring from postgres");
-					restore_from_postgres();
-				}
+			if( !wasCreated && postgresBackup.wasCreated ) {
+				luanLogger.error("rebuilding postgres backup");
+				rebuild_postgres_backup(completer);
+/*
+			} else if( wasCreated && !postgresBackup.wasCreated ) {
+				luanLogger.error("restoring from postgres");
+				restore_from_postgres();
+*/
 			}
 		}
+		luan.onClose(this);
+	}
+
+	protected void finalize() throws Throwable {
+		if( !isClosed ) {
+			sysLogger.error("not closed",created);
+			close();
+		}
+		super.finalize();
 	}
 
 	public boolean reopen() throws IOException {
@@ -449,15 +434,13 @@
 		return writer.getDirectory().toString();
 	}
 
-	private synchronized void close() throws IOException, SQLException {
-		if( openCount > 0 ) {
-			if( --openCount == 0 ) {
-				doClose();
-				synchronized(indexes) {
-					indexes.remove(key);
-				}
-			}
+	public synchronized void close() throws IOException {
+		try {
+			doClose();
+		} catch(SQLException e) {
+			throw new RuntimeException(e);
 		}
+		isClosed = true;
 	}
 
 	public void doClose() throws IOException, SQLException {
@@ -467,6 +450,15 @@
 			postgresBackup.close();
 	}
 
+	private void closeWriter() throws IOException {
+		writeLock.lock();
+		try {
+			writer.close();
+		} finally {
+			writeLock.unlock();
+		}
+	}
+
 
 	private static class DocFn extends LuanFunction {
 		final IndexSearcher searcher;
@@ -865,6 +857,16 @@
 	public void restore_from_postgres()
 		throws IOException, LuanException, SQLException
 	{
+		if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
+			luanLogger.error("restoring from postgres");
+			force_restore_from_postgres();
+			wasCreated = false;
+		}
+	}
+
+	public void force_restore_from_postgres()
+		throws IOException, LuanException, SQLException
+	{
 		luanLogger.warn("start restore_from_postgres");
 		if( postgresBackup==null )
 			throw new NullPointerException();