Mercurial Hosting > luan
changeset 1549:41c32da4cbd1
lucene log work
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sat, 03 Oct 2020 20:55:26 -0600 |
parents | 736ec76bbf42 |
children | 0dc3be25ad20 |
files | src/goodjava/lucene/api/GoodIndexWriter.java src/goodjava/lucene/api/GoodWriter.java src/goodjava/lucene/logging/FilterGoodWriter.java src/goodjava/lucene/logging/LoggingIndexWriter.java src/goodjava/lucene/logging/OpDoer.java |
diffstat | 5 files changed, 86 insertions(+), 94 deletions(-) [+] |
line wrap: on
line diff
--- a/src/goodjava/lucene/api/GoodIndexWriter.java Sun Sep 27 22:07:18 2020 -0600 +++ b/src/goodjava/lucene/api/GoodIndexWriter.java Sat Oct 03 20:55:26 2020 -0600 @@ -1,11 +1,20 @@ package goodjava.lucene.api; import java.io.IOException; +import java.util.Map; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.Query; -public interface GoodIndexWriter extends GoodWriter { +public interface GoodIndexWriter { + public IndexReader openReader() throws IOException; + public void commit() throws IOException; + public void deleteAll() throws IOException; + public void deleteDocuments(Query query) throws IOException; + public void addDocument(Map<String,Object> storedFields) throws IOException; + public void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException; + public void tag(String tag) throws IOException; public void close() throws IOException; public void rollback() throws IOException; public void reindexDocuments(String keyFieldName,Query query) throws IOException;
--- a/src/goodjava/lucene/api/GoodWriter.java Sun Sep 27 22:07:18 2020 -0600 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ -package goodjava.lucene.api; - -import java.io.IOException; -import java.util.Map; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.Query; - - -public interface GoodWriter { - public IndexReader openReader() throws IOException; - public void commit() throws IOException; - public void deleteAll() throws IOException; - public void deleteDocuments(Query query) throws IOException; - public void addDocument(Map<String,Object> storedFields) throws IOException; - public void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException; - public void tag(String tag) throws IOException; -}
--- a/src/goodjava/lucene/logging/FilterGoodWriter.java Sun Sep 27 22:07:18 2020 -0600 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,54 +0,0 @@ -package goodjava.lucene.logging; - -import java.io.IOException; -import java.util.Map; -import org.apache.lucene.search.Query; -import org.apache.lucene.index.IndexReader; -import goodjava.lucene.api.GoodWriter; - - -public class FilterGoodWriter implements GoodWriter { - protected final GoodWriter writer; - protected boolean isActive = true; - - protected FilterGoodWriter(GoodWriter writer) { - this.writer = writer; - } - - public IndexReader openReader() throws IOException { - return writer.openReader(); - } - - public void commit() throws IOException { - if( !isActive ) - return; - writer.commit(); - } - - public void deleteAll() throws IOException { - if( !isActive ) - return; - writer.deleteAll(); - } - - public void deleteDocuments(Query query) throws IOException { - if( !isActive ) - return; - writer.deleteDocuments(query); - } - - public void addDocument(Map<String,Object> storedFields) throws IOException { - if( !isActive ) - return; - writer.addDocument(storedFields); - } - - public void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { - if( !isActive ) - return; - writer.updateDocument(keyFieldName,storedFields); - } - - public void tag(String tag) throws IOException {} - -}
--- a/src/goodjava/lucene/logging/LoggingIndexWriter.java Sun Sep 27 22:07:18 2020 -0600 +++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java Sat Oct 03 20:55:26 2020 -0600 @@ -27,7 +27,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import goodjava.io.IoUtils; -import goodjava.lucene.api.GoodWriter; import goodjava.lucene.api.GoodIndexWriter; import goodjava.lucene.api.LuceneIndexWriter; import goodjava.lucene.api.GoodCollector; @@ -224,8 +223,9 @@ throw new RuntimeException(); Directory dir = FSDirectory.open(dirFile); LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); - playLog( first.input(), mergeWriter ); - playLog( second.input(), mergeWriter ); + OpDoer opDoer = new OpDoer(mergeWriter); + playLog( first.input(), opDoer ); + playLog( second.input(), opDoer ); mergeWriter.commit(); LogFile merge = newLogFile(); LogOutputStream log = merge.output(); @@ -300,7 +300,7 @@ IoUtils.deleteRecursively(dirFile); Directory dir = FSDirectory.open(dirFile); LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); - playLogs(logReaders,checkWriter); + playLogs(logReaders,new OpDoer(checkWriter)); //logger.info("check lucene"); IndexReader checkReader = checkWriter.openReader(); int nCheck = checkReader.numDocs(); @@ -449,10 +449,10 @@ } // return whether stopped at tag - public synchronized void playLogs(GoodWriter writer) throws IOException { - if( writer == null ) - writer = indexWriter; - playLogs( logReaders(logs), writer ); + public synchronized void playLogs(OpDoer opDoer) throws IOException { + if( opDoer == null ) + opDoer = new OpDoer(indexWriter); + playLogs( logReaders(logs), opDoer ); } private static LogInputStream[] logReaders(LogFile[] logs) throws IOException { @@ -463,66 +463,66 @@ return logReaders; } - private static void playLogs(LogInputStream[] logReaders,GoodWriter indexWriter) + private static void playLogs(LogInputStream[] logReaders,OpDoer opDoer) throws IOException { - if( numDocs(indexWriter) != 0 ) + if( numDocs(opDoer.writer) != 0 ) throw new RuntimeException ("not empty"); for( LogInputStream reader : logReaders ) { - playLog(reader,indexWriter); + playLog(reader,opDoer); } - indexWriter.commit(); + opDoer.commit(); } - private static int numDocs(GoodWriter indexWriter) throws IOException { + private static int numDocs(GoodIndexWriter indexWriter) throws IOException { IndexReader reader = indexWriter.openReader(); int n = reader.numDocs(); reader.close(); return n; } - private static void playLog(LogInputStream in,GoodWriter indexWriter) + private static void playLog(LogInputStream in,OpDoer opDoer) throws IOException { while( in.available() > 0 ) { - playOp(in,indexWriter); + playOp(in,opDoer); } in.close(); } - private static void playOp(LogInputStream in,GoodWriter indexWriter) + private static void playOp(LogInputStream in,OpDoer opDoer) throws IOException { - in.readLong(); // time + long time = in.readLong(); // time int op = in.readByte(); switch(op) { case OP_DELETE_ALL: - indexWriter.deleteAll(); + opDoer.deleteAll(time); return; case OP_DELETE_DOCUMENTS: { Query query = in.readQuery(); //System.out.println("OP_DELETE_DOCUMENTS "+query); - indexWriter.deleteDocuments(query); + opDoer.deleteDocuments(time,query); return; } case OP_ADD_DOCUMENT: { Map storedFields = in.readMap(); - indexWriter.addDocument(storedFields); + opDoer.addDocument(time,storedFields); return; } case OP_UPDATE_DOCUMENT: { String keyFieldName = in.readUTF(); Map storedFields = in.readMap(); - indexWriter.updateDocument(keyFieldName,storedFields); + opDoer.updateDocument(time,keyFieldName,storedFields); return; } case OP_TAG: { String tag = in.readUTF(); - indexWriter.tag(tag); + opDoer.tag(time,tag); return; } default:
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/goodjava/lucene/logging/OpDoer.java Sat Oct 03 20:55:26 2020 -0600 @@ -0,0 +1,54 @@ +package goodjava.lucene.logging; + +import java.io.IOException; +import java.util.Map; +import org.apache.lucene.search.Query; +import org.apache.lucene.index.IndexReader; +import goodjava.lucene.api.GoodIndexWriter; + + +public class OpDoer { + protected final GoodIndexWriter writer; + protected boolean isActive = true; + + protected OpDoer(GoodIndexWriter writer) { + this.writer = writer; + } + + public IndexReader openReader() throws IOException { + return writer.openReader(); + } + + public void commit() throws IOException { + if( !isActive ) + return; + writer.commit(); + } + + public void deleteAll(long time) throws IOException { + if( !isActive ) + return; + writer.deleteAll(); + } + + public void deleteDocuments(long time,Query query) throws IOException { + if( !isActive ) + return; + writer.deleteDocuments(query); + } + + public void addDocument(long time,Map<String,Object> storedFields) throws IOException { + if( !isActive ) + return; + writer.addDocument(storedFields); + } + + public void updateDocument(long time,String keyFieldName,Map<String,Object> storedFields) throws IOException { + if( !isActive ) + return; + writer.updateDocument(keyFieldName,storedFields); + } + + public void tag(long time,String tag) throws IOException {} + +}