Mercurial Hosting > luan
changeset 1556:52241b69c339
lucene logging
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 11 Oct 2020 17:28:16 -0600 |
parents | 1b397a949791 |
children | 117ce8645b7f |
files | src/goodjava/lucene/logging/BasicOpDoer.java src/goodjava/lucene/logging/LoggingIndexWriter.java src/goodjava/lucene/logging/OpDoer.java src/luan/modules/lucene/LuanOpDoer.java src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java |
diffstat | 6 files changed, 35 insertions(+), 54 deletions(-) [+] |
line wrap: on
line diff
diff -r 1b397a949791 -r 52241b69c339 src/goodjava/lucene/logging/BasicOpDoer.java --- a/src/goodjava/lucene/logging/BasicOpDoer.java Tue Oct 06 20:53:43 2020 -0600 +++ b/src/goodjava/lucene/logging/BasicOpDoer.java Sun Oct 11 17:28:16 2020 -0600 @@ -14,10 +14,6 @@ this.writer = writer; } - public GoodIndexWriter writer() { - return writer; - } - public void commit() throws IOException { if( !isActive ) return;
diff -r 1b397a949791 -r 52241b69c339 src/goodjava/lucene/logging/LoggingIndexWriter.java --- a/src/goodjava/lucene/logging/LoggingIndexWriter.java Tue Oct 06 20:53:43 2020 -0600 +++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java Sun Oct 11 17:28:16 2020 -0600 @@ -81,7 +81,16 @@ dis.close(); } } - newLogs(); + logger.info("building new logs"); + for( int i=0; i<logs.length; i++ ) { + logs[i] = newLogFile(); + } + LogOutputStream log = logs[0].output(); + logLucene( System.currentTimeMillis(), log, indexWriter ); + log.close(); + writeIndex(); + setLog(); + logger.info("done building new logs"); wasCreated = true; } @@ -112,32 +121,9 @@ } } - public synchronized void newLogs() throws IOException { - getMergeLock(); - try { - newLogs2(); - } finally { - mergeLock.unlock(); - } - } - - private void newLogs2() throws IOException { - logger.info("building new logs"); - for( int i=0; i<logs.length; i++ ) { - logs[i] = newLogFile(); - } - LogOutputStream log = logs[0].output(); - logLucene( System.currentTimeMillis(), log, indexWriter ); - log.close(); - writeIndex(); - setLog(); - logger.info("done building new logs"); - } - public synchronized void logLucene() throws IOException { - //log.rollback(); ? logLucene( System.currentTimeMillis(), log, indexWriter ); } @@ -465,21 +451,12 @@ private static void playLogs(LogInputStream[] logReaders,OpDoer opDoer) throws IOException { - if( numDocs(opDoer.writer()) != 0 ) - throw new RuntimeException ("not empty"); for( LogInputStream reader : logReaders ) { playLog(reader,opDoer); } opDoer.commit(); } - private static int numDocs(GoodIndexWriter indexWriter) throws IOException { - IndexReader reader = indexWriter.openReader(); - int n = reader.numDocs(); - reader.close(); - return n; - } - private static void playLog(LogInputStream in,OpDoer opDoer) throws IOException {
diff -r 1b397a949791 -r 52241b69c339 src/goodjava/lucene/logging/OpDoer.java --- a/src/goodjava/lucene/logging/OpDoer.java Tue Oct 06 20:53:43 2020 -0600 +++ b/src/goodjava/lucene/logging/OpDoer.java Sun Oct 11 17:28:16 2020 -0600 @@ -7,7 +7,6 @@ public interface OpDoer { - public GoodIndexWriter writer(); public void commit() throws IOException; public void deleteAll(long time) throws IOException; public void deleteDocuments(long time,Query query) throws IOException;
diff -r 1b397a949791 -r 52241b69c339 src/luan/modules/lucene/LuanOpDoer.java --- a/src/luan/modules/lucene/LuanOpDoer.java Tue Oct 06 20:53:43 2020 -0600 +++ b/src/luan/modules/lucene/LuanOpDoer.java Sun Oct 11 17:28:16 2020 -0600 @@ -20,10 +20,6 @@ this.fn = fn; } - public GoodIndexWriter writer() { - return opDoer.writer(); - } - public void commit() throws IOException { try { fn.call(new CommitAction(opDoer));
diff -r 1b397a949791 -r 52241b69c339 src/luan/modules/lucene/Lucene.luan --- a/src/luan/modules/lucene/Lucene.luan Tue Oct 06 20:53:43 2020 -0600 +++ b/src/luan/modules/lucene/Lucene.luan Sun Oct 11 17:28:16 2020 -0600 @@ -76,13 +76,14 @@ index.count_tokens = java_index.count_tokens index.tag = java_index.tag - index.rebuild_log = java_index.rebuild_log + index.relog = java_index.relog index.restore_from_log = java_index.restore_from_log + index.force_restore_from_log = java_index.force_restore_from_log index.has_postgres_backup = java_index.hasPostgresBackup() index.rebuild_postgres_backup = java_index.rebuild_postgres_backup index.restore_from_postgres = java_index.restore_from_postgres - --index.force_restore_from_postgres = java_index.force_restore_from_postgres + index.force_restore_from_postgres = java_index.force_restore_from_postgres index.check = java_index.check function index.not_in_transaction() end
diff -r 1b397a949791 -r 52241b69c339 src/luan/modules/lucene/LuceneIndex.java --- a/src/luan/modules/lucene/LuceneIndex.java Tue Oct 06 20:53:43 2020 -0600 +++ b/src/luan/modules/lucene/LuceneIndex.java Sun Oct 11 17:28:16 2020 -0600 @@ -421,7 +421,14 @@ } public void tag(String tag) throws IOException { - writer.tag(tag); + boolean commit = !writeLock.isHeldByCurrentThread(); + writeLock.lock(); + try { + writer.tag(tag); + if(commit) writer.commit(); + } finally { + writeLock.unlock(); + } } @@ -770,6 +777,20 @@ writer.addDocument(toLucene(doc)); } + public void relog() + throws IOException, LuanException + { + logger.info("start relog"); + writeLock.lock(); + try { + LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; + loggingWriter.logLucene(); + } finally { + writeLock.unlock(); + } + logger.info("end relog"); + } + public void restore_from_log(LuanFunction handler) throws IOException, LuanException, SQLException, ParseException { @@ -791,8 +812,6 @@ boolean ok = false; try { LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; - IndexWriter iw = writer.getLuceneIndexWriter(); - iw.deleteAll(); loggingWriter.playLogs(opDoer); ok = true; wrote(); @@ -831,13 +850,6 @@ logger.info("end check"); } - public void rebuild_log() throws IOException { - logger.info("start rebuild_log"); - LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; - loggingWriter.newLogs(); - logger.info("end rebuild_log"); - } - private void checkPostgres(Luan luan) throws IOException, SQLException, LuanException, ParseException {