Mercurial Hosting > luan

comparison src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1484:1fa6e8ec2d53

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene.logging cleanup
author Franklin Schmidt <fschmidt@gmail.com>
date Sat, 25 Apr 2020 11:14:25 -0600
parents 7d145095cc0b
children 2469aa31f31b
comparison
equal deleted inserted replaced
1483:97740900c820 1484:1fa6e8ec2d53
62 try { 62 try {
63 if( dis.readInt() == version ) { 63 if( dis.readInt() == version ) {
64 final int n = dis.readInt(); 64 final int n = dis.readInt();
65 for( int i=0; i<n; i++ ) { 65 for( int i=0; i<n; i++ ) {
66 File file = new File( logDir, dis.readUTF() ); 66 File file = new File( logDir, dis.readUTF() );
67 logs.add( new LogFile(file,"rwd") ); 67 logs.add( LogFile.newLogFile(file) );
68 } 68 }
69 deleteUnusedFiles(); 69 deleteUnusedFiles();
70 return; 70 return;
71 } 71 }
72 } finally { 72 } finally {
117 private LogFile newLogFile() throws IOException { 117 private LogFile newLogFile() throws IOException {
118 File file; 118 File file;
119 do { 119 do {
120 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); 120 file = new File(logDir,"_"+rnd.nextInt(100)+".log");
121 } while( file.exists() ); 121 } while( file.exists() );
122 return new LogFile(file,"rwd"); 122 return LogFile.newLogFile(file);
123 } 123 }
124 124
125 private void deleteUnusedFiles() throws IOException { 125 private void deleteUnusedFiles() throws IOException {
126 Set<String> used = new HashSet<String>(); 126 Set<String> used = new HashSet<String>();
127 used.add( index.getName() ); 127 used.add( index.getName() );
160 File dirFile = new File(logDir,"merge"); 160 File dirFile = new File(logDir,"merge");
161 if( dirFile.exists() ) 161 if( dirFile.exists() )
162 throw new RuntimeException(); 162 throw new RuntimeException();
163 Directory dir = FSDirectory.open(dirFile); 163 Directory dir = FSDirectory.open(dirFile);
164 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig ); 164 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig );
165 playLog(first,mergeWriter); 165 playLog( first.input(), mergeWriter );
166 playLog(second,mergeWriter); 166 playLog( second.input(), mergeWriter );
167 mergeWriter.commit(); 167 mergeWriter.commit();
168 LogFile merge = newLogFile(); 168 LogFile merge = newLogFile();
169 logLucene( lastTime, merge, mergeWriter ); 169 logLucene( lastTime, merge, mergeWriter );
170 mergeWriter.close(); 170 mergeWriter.close();
171 synchronized(this) { 171 synchronized(this) {
206 } 206 }
207 } 207 }
208 208
209 public void check(SortField sortField) throws IOException { 209 public void check(SortField sortField) throws IOException {
210 IndexReader indexReader; 210 IndexReader indexReader;
211 List<LogFile> logs; 211 List<LogInputStream> logReaders;
212 synchronized(this) { 212 synchronized(this) {
213 if( isMerging ) { 213 if( isMerging ) {
214 logger.warn("is merging, check aborted"); 214 logger.warn("is merging, check aborted");
215 return; 215 return;
216 } 216 }
217 isMerging = true; 217 isMerging = true;
218 indexReader = indexWriter.openReader(); 218 indexReader = indexWriter.openReader();
219 logs = new ArrayList<LogFile>(this.logs); 219 logReaders = logReaders(logs);
220 int i = logs.size() - 1;
221 LogFile last = logs.get(i);
222 logs.set(i,last.snapshot());
223 } 220 }
224 try { 221 try {
225 logger.info("check start"); 222 logger.info("check start");
226 indexWriter.check(); 223 indexWriter.check();
227 File dirFile = new File(logDir,"check"); 224 File dirFile = new File(logDir,"check");
228 IoUtils.deleteRecursively(dirFile); 225 IoUtils.deleteRecursively(dirFile);
229 Directory dir = FSDirectory.open(dirFile); 226 Directory dir = FSDirectory.open(dirFile);
230 LuceneIndexWriter checkWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig ); 227 LuceneIndexWriter checkWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig );
231 playLogs(logs,checkWriter); 228 playLogs(logReaders,checkWriter);
232 logger.info("check lucene"); 229 logger.info("check lucene");
233 IndexReader checkReader = checkWriter.openReader(); 230 IndexReader checkReader = checkWriter.openReader();
234 if( sortField == null ) { 231 if( sortField == null ) {
235 int nCheck = checkReader.numDocs(); 232 int nCheck = checkReader.numDocs();
236 int nOrig = indexReader.numDocs(); 233 int nOrig = indexReader.numDocs();
378 log.writeLong(System.currentTimeMillis()); 375 log.writeLong(System.currentTimeMillis());
379 log.writeByte(op); 376 log.writeByte(op);
380 } 377 }
381 378
382 public synchronized void playLogs() throws IOException { 379 public synchronized void playLogs() throws IOException {
383 playLogs(logs,indexWriter); 380 playLogs( logReaders(logs), indexWriter );
384 } 381 }
385 382
386 private static void playLogs(List<LogFile> logs,LuceneIndexWriter indexWriter) throws IOException { 383 private static List<LogInputStream> logReaders(List<LogFile> logs) throws IOException {
384 List<LogInputStream> logReaders = new ArrayList<LogInputStream>();
385 for( LogFile log : logs ) {
386 logReaders.add( log.input() );
387 }
388 return logReaders;
389 }
390
391 private static void playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter) throws IOException {
387 if( numDocs(indexWriter) != 0 ) 392 if( numDocs(indexWriter) != 0 )
388 throw new RuntimeException ("not empty"); 393 throw new RuntimeException ("not empty");
389 for( LogFile log : logs ) { 394 for( LogInputStream reader : logReaders ) {
390 playLog(log,indexWriter); 395 playLog(reader,indexWriter);
391 } 396 }
392 indexWriter.commit(); 397 indexWriter.commit();
393 } 398 }
394 399
395 private static int numDocs(LuceneIndexWriter indexWriter) throws IOException { 400 private static int numDocs(LuceneIndexWriter indexWriter) throws IOException {
397 int n = reader.numDocs(); 402 int n = reader.numDocs();
398 reader.close(); 403 reader.close();
399 return n; 404 return n;
400 } 405 }
401 406
402 private static void playLog(LogFile log,LuceneIndexWriter indexWriter) throws IOException { 407 private static void playLog(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException {
403 LogInputStream in = log.input();
404 while( in.available() > 0 ) { 408 while( in.available() > 0 ) {
405 playOp(in,indexWriter); 409 playOp(in,indexWriter);
406 } 410 }
411 in.close();
407 } 412 }
408 413
409 private static void playOp(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException { 414 private static void playOp(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException {
410 in.readLong(); // time 415 in.readLong(); // time
411 int op = in.readByte(); 416 int op = in.readByte();