changeset 1556:52241b69c339

lucene logging
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 11 Oct 2020 17:28:16 -0600
parents 1b397a949791
children 117ce8645b7f
files src/goodjava/lucene/logging/BasicOpDoer.java src/goodjava/lucene/logging/LoggingIndexWriter.java src/goodjava/lucene/logging/OpDoer.java src/luan/modules/lucene/LuanOpDoer.java src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java
diffstat 6 files changed, 35 insertions(+), 54 deletions(-) [+]
line wrap: on
line diff
diff -r 1b397a949791 -r 52241b69c339 src/goodjava/lucene/logging/BasicOpDoer.java
--- a/src/goodjava/lucene/logging/BasicOpDoer.java	Tue Oct 06 20:53:43 2020 -0600
+++ b/src/goodjava/lucene/logging/BasicOpDoer.java	Sun Oct 11 17:28:16 2020 -0600
@@ -14,10 +14,6 @@
 		this.writer = writer;
 	}
 
-	public GoodIndexWriter writer() {
-		return writer;
-	}
-
 	public void commit() throws IOException {
 		if( !isActive )
 			return;
diff -r 1b397a949791 -r 52241b69c339 src/goodjava/lucene/logging/LoggingIndexWriter.java
--- a/src/goodjava/lucene/logging/LoggingIndexWriter.java	Tue Oct 06 20:53:43 2020 -0600
+++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sun Oct 11 17:28:16 2020 -0600
@@ -81,7 +81,16 @@
 				dis.close();
 			}
 		}
-		newLogs();
+		logger.info("building new logs");
+		for( int i=0; i<logs.length; i++ ) {
+			logs[i] = newLogFile();
+		}
+		LogOutputStream log = logs[0].output();
+		logLucene( System.currentTimeMillis(), log, indexWriter );
+		log.close();
+		writeIndex();
+		setLog();
+		logger.info("done building new logs");
 		wasCreated = true;
 	}
 
@@ -112,32 +121,9 @@
 		}
 	}
 
-	public synchronized void newLogs() throws IOException {
-		getMergeLock();
-		try {
-			newLogs2();
-		} finally {
-			mergeLock.unlock();
-		}
-	}
-
-	private void newLogs2() throws IOException {
-		logger.info("building new logs");
-		for( int i=0; i<logs.length; i++ ) {
-			logs[i] = newLogFile();
-		}
-		LogOutputStream log = logs[0].output();
-		logLucene( System.currentTimeMillis(), log, indexWriter );
-		log.close();
-		writeIndex();
-		setLog();
-		logger.info("done building new logs");
-	}
-
 	public synchronized void logLucene()
 		throws IOException
 	{
-		//log.rollback();  ?
 		logLucene( System.currentTimeMillis(), log, indexWriter );
 	}
 
@@ -465,21 +451,12 @@
 	private static void playLogs(LogInputStream[] logReaders,OpDoer opDoer)
 		throws IOException
 	{
-		if( numDocs(opDoer.writer()) != 0 )
-			throw new RuntimeException ("not empty");
 		for( LogInputStream reader : logReaders ) {
 			playLog(reader,opDoer);
 		}
 		opDoer.commit();
 	}
 
-	private static int numDocs(GoodIndexWriter indexWriter) throws IOException {
-		IndexReader reader = indexWriter.openReader();
-		int n = reader.numDocs();
-		reader.close();
-		return n;
-	}
-
 	private static void playLog(LogInputStream in,OpDoer opDoer)
 		throws IOException
 	{
diff -r 1b397a949791 -r 52241b69c339 src/goodjava/lucene/logging/OpDoer.java
--- a/src/goodjava/lucene/logging/OpDoer.java	Tue Oct 06 20:53:43 2020 -0600
+++ b/src/goodjava/lucene/logging/OpDoer.java	Sun Oct 11 17:28:16 2020 -0600
@@ -7,7 +7,6 @@
 
 
 public interface OpDoer {
-	public GoodIndexWriter writer();
 	public void commit() throws IOException;
 	public void deleteAll(long time) throws IOException;
 	public void deleteDocuments(long time,Query query) throws IOException;
diff -r 1b397a949791 -r 52241b69c339 src/luan/modules/lucene/LuanOpDoer.java
--- a/src/luan/modules/lucene/LuanOpDoer.java	Tue Oct 06 20:53:43 2020 -0600
+++ b/src/luan/modules/lucene/LuanOpDoer.java	Sun Oct 11 17:28:16 2020 -0600
@@ -20,10 +20,6 @@
 		this.fn = fn;
 	}
 
-	public GoodIndexWriter writer() {
-		return opDoer.writer();
-	}
-
 	public void commit() throws IOException {
 		try {
 			fn.call(new CommitAction(opDoer));
diff -r 1b397a949791 -r 52241b69c339 src/luan/modules/lucene/Lucene.luan
--- a/src/luan/modules/lucene/Lucene.luan	Tue Oct 06 20:53:43 2020 -0600
+++ b/src/luan/modules/lucene/Lucene.luan	Sun Oct 11 17:28:16 2020 -0600
@@ -76,13 +76,14 @@
 	index.count_tokens = java_index.count_tokens
 	index.tag = java_index.tag
 
-	index.rebuild_log = java_index.rebuild_log
+	index.relog = java_index.relog
 	index.restore_from_log = java_index.restore_from_log
+	index.force_restore_from_log = java_index.force_restore_from_log
 
 	index.has_postgres_backup = java_index.hasPostgresBackup()
 	index.rebuild_postgres_backup = java_index.rebuild_postgres_backup
 	index.restore_from_postgres = java_index.restore_from_postgres
-	--index.force_restore_from_postgres = java_index.force_restore_from_postgres
+	index.force_restore_from_postgres = java_index.force_restore_from_postgres
 	index.check = java_index.check
 
 	function index.not_in_transaction() end
diff -r 1b397a949791 -r 52241b69c339 src/luan/modules/lucene/LuceneIndex.java
--- a/src/luan/modules/lucene/LuceneIndex.java	Tue Oct 06 20:53:43 2020 -0600
+++ b/src/luan/modules/lucene/LuceneIndex.java	Sun Oct 11 17:28:16 2020 -0600
@@ -421,7 +421,14 @@
 	}
 
 	public void tag(String tag) throws IOException {
-		writer.tag(tag);
+		boolean commit = !writeLock.isHeldByCurrentThread();
+		writeLock.lock();
+		try {
+			writer.tag(tag);
+			if(commit) writer.commit();
+		} finally {
+			writeLock.unlock();
+		}
 	}
 
 
@@ -770,6 +777,20 @@
 		writer.addDocument(toLucene(doc));
 	}
 
+	public void relog()
+		throws IOException, LuanException
+	{
+		logger.info("start relog");
+		writeLock.lock();
+		try {
+			LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
+			loggingWriter.logLucene();
+		} finally {
+			writeLock.unlock();
+		}
+		logger.info("end relog");
+	}
+
 	public void restore_from_log(LuanFunction handler)
 		throws IOException, LuanException, SQLException, ParseException
 	{
@@ -791,8 +812,6 @@
 		boolean ok = false;
 		try {
 			LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
-			IndexWriter iw = writer.getLuceneIndexWriter();
-			iw.deleteAll();
 			loggingWriter.playLogs(opDoer);
 			ok = true;
 			wrote();
@@ -831,13 +850,6 @@
 		logger.info("end check");
 	}
 
-	public void rebuild_log() throws IOException {
-		logger.info("start rebuild_log");
-		LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
-		loggingWriter.newLogs();
-		logger.info("end rebuild_log");
-	}
-
 	private void checkPostgres(Luan luan)
 		throws IOException, SQLException, LuanException, ParseException
 	{