Mercurial Hosting > luan
diff src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1544:35601f15ecc3
add lucene log tag and restore_from_log
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 20 Sep 2020 20:36:55 -0600 |
parents | c27dc6af87ca |
children | 736ec76bbf42 |
line wrap: on
line diff
--- a/src/goodjava/lucene/logging/LoggingIndexWriter.java Sun Sep 20 16:45:31 2020 -0600 +++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java Sun Sep 20 20:36:55 2020 -0600 @@ -44,9 +44,11 @@ private static final int OP_DELETE_DOCUMENTS = 2; private static final int OP_ADD_DOCUMENT = 3; private static final int OP_UPDATE_DOCUMENT = 4; + private static final int OP_TAG = 5; private static final Random rnd = new Random(); public final LuceneIndexWriter indexWriter; + public boolean wasCreated; private final File logDir; protected final List<LogFile> logs = new ArrayList<LogFile>(); private LogOutputStream log; @@ -71,6 +73,7 @@ } deleteUnusedFiles(); setLog(); + wasCreated = false; return; } } finally { @@ -78,6 +81,7 @@ } } newLogs(); + wasCreated = true; } public IndexWriter getLuceneIndexWriter() { @@ -201,8 +205,8 @@ throw new RuntimeException(); Directory dir = FSDirectory.open(dirFile); LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); - playLog( first.input(), mergeWriter ); - playLog( second.input(), mergeWriter ); + playLog( first.input(), mergeWriter, null ); + playLog( second.input(), mergeWriter, null ); mergeWriter.commit(); LogFile merge = newLogFile(); logLucene( lastTime, merge, mergeWriter ); @@ -273,18 +277,17 @@ IoUtils.deleteRecursively(dirFile); Directory dir = FSDirectory.open(dirFile); LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); - playLogs(logReaders,checkWriter); + playLogs(logReaders,checkWriter,null); //logger.info("check lucene"); IndexReader checkReader = checkWriter.openReader(); + int nCheck = checkReader.numDocs(); + int nOrig = indexReader.numDocs(); + if( nCheck != nOrig ) { + logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); + ok = false; + } if( sortField == null ) { - int nCheck = checkReader.numDocs(); - int nOrig = indexReader.numDocs(); - if( nCheck != nOrig ) { - logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); - ok = false; - } - //logger.info("numDocs="+nOrig); - if( hash(indexReader) != hash(checkReader) ) { + if( ok && hash(indexReader) != hash(checkReader) ) { logger.error("hash mismatch"); ok = false; } @@ -413,6 +416,11 @@ log.writeMap(storedFields); } + public synchronized void tag(String tag) throws IOException { + writeOp(OP_TAG); + log.writeUTF(tag); + } + public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { indexWriter.reindexDocuments(keyFieldName,query); } @@ -422,8 +430,9 @@ log.writeByte(op); } - public synchronized void playLogs() throws IOException { - playLogs( logReaders(logs), indexWriter ); + // return whether stopped at tag + public synchronized boolean playLogs(String upToTag) throws IOException { + return playLogs( logReaders(logs), indexWriter, upToTag ); } private static List<LogInputStream> logReaders(List<LogFile> logs) throws IOException { @@ -434,13 +443,20 @@ return logReaders; } - private static void playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter) throws IOException { + private static boolean playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter,String upToTag) + throws IOException + { if( numDocs(indexWriter) != 0 ) throw new RuntimeException ("not empty"); + boolean rtn = false; for( LogInputStream reader : logReaders ) { - playLog(reader,indexWriter); + if( playLog(reader,indexWriter,upToTag) ) { + rtn = true; + break; + } } indexWriter.commit(); + return rtn; } private static int numDocs(LuceneIndexWriter indexWriter) throws IOException { @@ -450,35 +466,51 @@ return n; } - private static void playLog(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException { + private static boolean playLog(LogInputStream in,LuceneIndexWriter indexWriter,String upToTag) + throws IOException + { + boolean rtn = false; while( in.available() > 0 ) { - playOp(in,indexWriter); + if( playOp(in,indexWriter,upToTag) ) { + rtn = true; + break; + } } in.close(); + return rtn; } - private static void playOp(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException { + private static boolean playOp(LogInputStream in,LuceneIndexWriter indexWriter,String upToTag) throws IOException { in.readLong(); // time int op = in.readByte(); switch(op) { case OP_DELETE_ALL: indexWriter.deleteAll(); - return; + return false; case OP_DELETE_DOCUMENTS: - indexWriter.deleteDocuments( in.readQuery() ); - return; + { + Query query = in.readQuery(); + //System.out.println("OP_DELETE_DOCUMENTS "+query); + indexWriter.deleteDocuments(query); + return false; + } case OP_ADD_DOCUMENT: { Map storedFields = in.readMap(); indexWriter.addDocument(storedFields); - return; + return false; } case OP_UPDATE_DOCUMENT: { String keyFieldName = in.readUTF(); Map storedFields = in.readMap(); indexWriter.updateDocument(keyFieldName,storedFields); - return; + return false; + } + case OP_TAG: + { + String tag = in.readUTF(); + return tag.equals(upToTag); } default: throw new RuntimeException("invalid op "+op);