Mercurial Hosting > luan

comparison src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1486:2469aa31f31b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
LogOutputStream
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 01 May 2020 16:09:35 -0600
parents 1fa6e8ec2d53
children 044a360c2300
comparison
equal deleted inserted replaced
1485:6a24c8b33d6b 1486:2469aa31f31b
45 private static final Random rnd = new Random(); 45 private static final Random rnd = new Random();
46 46
47 public final LuceneIndexWriter indexWriter; 47 public final LuceneIndexWriter indexWriter;
48 private final File logDir; 48 private final File logDir;
49 private final List<LogFile> logs = new ArrayList<LogFile>(); 49 private final List<LogFile> logs = new ArrayList<LogFile>();
50 private LogOutputStream log;
50 private final File index; 51 private final File index;
51 private boolean isMerging = false; 52 private boolean isMerging = false;
52 53
53 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir) throws IOException { 54 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir) throws IOException {
54 this.indexWriter = indexWriter; 55 this.indexWriter = indexWriter;
62 try { 63 try {
63 if( dis.readInt() == version ) { 64 if( dis.readInt() == version ) {
64 final int n = dis.readInt(); 65 final int n = dis.readInt();
65 for( int i=0; i<n; i++ ) { 66 for( int i=0; i<n; i++ ) {
66 File file = new File( logDir, dis.readUTF() ); 67 File file = new File( logDir, dis.readUTF() );
67 logs.add( LogFile.newLogFile(file) ); 68 logs.add( new LogFile(file) );
68 } 69 }
69 deleteUnusedFiles(); 70 deleteUnusedFiles();
71 setLog();
70 return; 72 return;
71 } 73 }
72 } finally { 74 } finally {
73 dis.close(); 75 dis.close();
74 } 76 }
75 } 77 }
76 newLogs(); 78 newLogs();
79 }
80
81 private void setLog() throws IOException {
82 if( log != null )
83 log.close();
84 log = logs.get(logs.size()-1).output();
77 } 85 }
78 86
79 public synchronized boolean isMerging() { 87 public synchronized boolean isMerging() {
80 return isMerging; 88 return isMerging;
81 } 89 }
92 for( int i=0; i<2; i++ ) { 100 for( int i=0; i<2; i++ ) {
93 logs.add( newLogFile() ); 101 logs.add( newLogFile() );
94 } 102 }
95 logLucene( System.currentTimeMillis(), logs.get(0), indexWriter ); 103 logLucene( System.currentTimeMillis(), logs.get(0), indexWriter );
96 writeIndex(); 104 writeIndex();
105 setLog();
97 logger.info("done building new logs"); 106 logger.info("done building new logs");
98 } 107 }
99 108
100 private static void logLucene(long time,LogFile log,LuceneIndexWriter indexWriter) throws IOException { 109 private static void logLucene(long time,LogFile logLucene,LuceneIndexWriter indexWriter) throws IOException {
110 LogOutputStream log = logLucene.output();
101 IndexReader reader = indexWriter.openReader(); 111 IndexReader reader = indexWriter.openReader();
102 final IndexSearcher searcher = new IndexSearcher(reader); 112 final IndexSearcher searcher = new IndexSearcher(reader);
103 Query query = new MatchAllDocsQuery(); 113 Query query = new MatchAllDocsQuery();
104 searcher.search( query, new GoodCollector(){ 114 searcher.search( query, new GoodCollector(){
105 public void collectDoc(int iDoc) throws IOException { 115 public void collectDoc(int iDoc) throws IOException {
110 log.writeMap(storedFields); 120 log.writeMap(storedFields);
111 } 121 }
112 }); 122 });
113 reader.close(); 123 reader.close();
114 log.commit(); 124 log.commit();
125 log.close();
115 } 126 }
116 127
117 private LogFile newLogFile() throws IOException { 128 private LogFile newLogFile() throws IOException {
118 File file; 129 File file;
119 do { 130 do {
120 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); 131 file = new File(logDir,"_"+rnd.nextInt(100)+".log");
121 } while( file.exists() ); 132 } while( file.exists() );
122 return LogFile.newLogFile(file); 133 return new LogFile(file);
123 } 134 }
124 135
125 private void deleteUnusedFiles() throws IOException { 136 private void deleteUnusedFiles() throws IOException {
126 Set<String> used = new HashSet<String>(); 137 Set<String> used = new HashSet<String>();
127 used.add( index.getName() ); 138 used.add( index.getName() );
304 }; 315 };
305 searcher.search(query,col); 316 searcher.search(query,col);
306 return col.total; 317 return col.total;
307 } 318 }
308 319
309 private LogFile log() {
310 return logs.get(logs.size()-1);
311 }
312
313 public synchronized void close() throws IOException { 320 public synchronized void close() throws IOException {
314 indexWriter.close(); 321 indexWriter.close();
315 LogFile log = log();
316 log.commit(); 322 log.commit();
323 log.close();
317 } 324 }
318 325
319 public synchronized void commit() throws IOException { 326 public synchronized void commit() throws IOException {
320 indexWriter.commit(); 327 indexWriter.commit();
321 LogFile log = log();
322 log.commit(); 328 log.commit();
323 if( isMerging ) 329 if( isMerging )
324 return; 330 return;
325 if( log.end() > logs.get(0).end() ) { 331 if( log.logFile.end() > logs.get(0).end() ) {
326 logs.add( newLogFile() ); 332 logs.add( newLogFile() );
327 writeIndex(); 333 writeIndex();
334 setLog();
328 } 335 }
329 if( logs.size() > 3 ) { 336 if( logs.size() > 3 ) {
330 isMerging = true; 337 isMerging = true;
331 new Thread(mergeLogs).start(); 338 new Thread(mergeLogs).start();
332 // mergeLogs.run(); 339 // mergeLogs.run();
333 } 340 }
334 } 341 }
335 342
336 public synchronized void rollback() throws IOException { 343 public synchronized void rollback() throws IOException {
337 indexWriter.rollback(); 344 indexWriter.rollback();
338 LogFile log = log();
339 log.rollback(); 345 log.rollback();
340 } 346 }
341 347
342 public synchronized void deleteAll() throws IOException { 348 public synchronized void deleteAll() throws IOException {
343 indexWriter.deleteAll(); 349 indexWriter.deleteAll();
344 LogFile log = log(); 350 writeOp(OP_DELETE_ALL);
345 writeOp(log,OP_DELETE_ALL);
346 } 351 }
347 352
348 public synchronized void deleteDocuments(Query query) throws IOException { 353 public synchronized void deleteDocuments(Query query) throws IOException {
349 indexWriter.deleteDocuments(query); 354 indexWriter.deleteDocuments(query);
350 LogFile log = log(); 355 writeOp(OP_DELETE_DOCUMENTS);
351 writeOp(log,OP_DELETE_DOCUMENTS);
352 log.writeQuery(query); 356 log.writeQuery(query);
353 } 357 }
354 358
355 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException { 359 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException {
356 indexWriter.addDocument(storedFields); 360 indexWriter.addDocument(storedFields);
357 LogFile log = log(); 361 writeOp(OP_ADD_DOCUMENT);
358 writeOp(log,OP_ADD_DOCUMENT);
359 log.writeMap(storedFields); 362 log.writeMap(storedFields);
360 } 363 }
361 364
362 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { 365 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException {
363 indexWriter.updateDocument(keyFieldName,storedFields); 366 indexWriter.updateDocument(keyFieldName,storedFields);
364 LogFile log = log(); 367 writeOp(OP_UPDATE_DOCUMENT);
365 writeOp(log,OP_UPDATE_DOCUMENT);
366 log.writeUTF(keyFieldName); 368 log.writeUTF(keyFieldName);
367 log.writeMap(storedFields); 369 log.writeMap(storedFields);
368 } 370 }
369 371
370 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { 372 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException {
371 indexWriter.reindexDocuments(keyFieldName,query); 373 indexWriter.reindexDocuments(keyFieldName,query);
372 } 374 }
373 375
374 private void writeOp(LogFile log,int op) throws IOException { 376 private void writeOp(int op) throws IOException {
375 log.writeLong(System.currentTimeMillis()); 377 log.writeLong(System.currentTimeMillis());
376 log.writeByte(op); 378 log.writeByte(op);
377 } 379 }
378 380
379 public synchronized void playLogs() throws IOException { 381 public synchronized void playLogs() throws IOException {