Mercurial Hosting > luan
annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1484:1fa6e8ec2d53
lucene.logging cleanup
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Sat, 25 Apr 2020 11:14:25 -0600 |
| parents | 7d145095cc0b |
| children | 2469aa31f31b |
| rev | line source |
|---|---|
| 1461 | 1 package goodjava.lucene.logging; |
| 2 | |
| 1465 | 3 import java.io.File; |
| 4 import java.io.RandomAccessFile; | |
| 5 import java.io.ByteArrayOutputStream; | |
| 6 import java.io.DataOutputStream; | |
| 7 import java.io.DataInputStream; | |
| 8 import java.io.FileInputStream; | |
| 1461 | 9 import java.io.IOException; |
| 10 import java.util.Map; | |
| 1465 | 11 import java.util.Set; |
| 12 import java.util.HashSet; | |
| 13 import java.util.List; | |
| 14 import java.util.ArrayList; | |
| 15 import java.util.Random; | |
| 16 import org.apache.lucene.document.Document; | |
| 17 import org.apache.lucene.index.DirectoryReader; | |
| 18 import org.apache.lucene.index.IndexReader; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
19 import org.apache.lucene.index.Term; |
| 1465 | 20 import org.apache.lucene.search.IndexSearcher; |
| 1461 | 21 import org.apache.lucene.search.Query; |
| 1465 | 22 import org.apache.lucene.search.MatchAllDocsQuery; |
| 23 import org.apache.lucene.search.TopDocs; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
24 import org.apache.lucene.search.PrefixQuery; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
25 import org.apache.lucene.search.SortField; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
26 import org.apache.lucene.search.Sort; |
| 1465 | 27 import org.apache.lucene.store.Directory; |
| 28 import org.apache.lucene.store.FSDirectory; | |
| 1473 | 29 import goodjava.io.IoUtils; |
| 1461 | 30 import goodjava.lucene.api.GoodIndexWriter; |
| 1465 | 31 import goodjava.lucene.api.LuceneIndexWriter; |
| 32 import goodjava.lucene.api.GoodCollector; | |
| 33 import goodjava.lucene.api.LuceneUtils; | |
| 34 import goodjava.logging.Logger; | |
| 35 import goodjava.logging.LoggerFactory; | |
| 1461 | 36 |
| 37 | |
| 1465 | 38 public final class LoggingIndexWriter implements GoodIndexWriter { |
| 39 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class); | |
| 40 private static final int version = 1; | |
| 1461 | 41 private static final int OP_DELETE_ALL = 1; |
| 42 private static final int OP_DELETE_DOCUMENTS = 2; | |
| 43 private static final int OP_ADD_DOCUMENT = 3; | |
| 44 private static final int OP_UPDATE_DOCUMENT = 4; | |
| 1465 | 45 private static final Random rnd = new Random(); |
| 1461 | 46 |
| 1465 | 47 public final LuceneIndexWriter indexWriter; |
| 48 private final File logDir; | |
| 49 private final List<LogFile> logs = new ArrayList<LogFile>(); | |
| 50 private final File index; | |
| 51 private boolean isMerging = false; | |
| 1461 | 52 |
| 1465 | 53 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir) throws IOException { |
| 1461 | 54 this.indexWriter = indexWriter; |
| 1465 | 55 this.logDir = logDir; |
| 56 logDir.mkdirs(); | |
| 57 if( !logDir.isDirectory() ) | |
| 58 throw new RuntimeException(); | |
| 59 index = new File(logDir,"index"); | |
| 60 if( index.exists() ) { | |
| 61 DataInputStream dis = new DataInputStream(new FileInputStream(index)); | |
| 62 try { | |
| 63 if( dis.readInt() == version ) { | |
| 64 final int n = dis.readInt(); | |
| 65 for( int i=0; i<n; i++ ) { | |
| 66 File file = new File( logDir, dis.readUTF() ); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
67 logs.add( LogFile.newLogFile(file) ); |
| 1465 | 68 } |
| 69 deleteUnusedFiles(); | |
| 70 return; | |
| 71 } | |
| 72 } finally { | |
| 73 dis.close(); | |
| 74 } | |
| 75 } | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
76 newLogs(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
77 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
78 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
79 public synchronized boolean isMerging() { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
80 return isMerging; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
81 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
82 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
83 private synchronized void isNotMerging() { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
84 isMerging = false; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
85 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
86 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
87 public synchronized void newLogs() throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
88 if( isMerging ) |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
89 throw new RuntimeException("merging"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
90 logger.info("building new logs"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
91 logs.clear(); |
| 1465 | 92 for( int i=0; i<2; i++ ) { |
| 93 logs.add( newLogFile() ); | |
| 94 } | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
95 logLucene( System.currentTimeMillis(), logs.get(0), indexWriter ); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
96 writeIndex(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
97 logger.info("done building new logs"); |
| 1461 | 98 } |
| 99 | |
| 1465 | 100 private static void logLucene(long time,LogFile log,LuceneIndexWriter indexWriter) throws IOException { |
| 101 IndexReader reader = indexWriter.openReader(); | |
| 102 final IndexSearcher searcher = new IndexSearcher(reader); | |
| 103 Query query = new MatchAllDocsQuery(); | |
| 104 searcher.search( query, new GoodCollector(){ | |
| 105 public void collectDoc(int iDoc) throws IOException { | |
| 106 Document doc = searcher.doc(iDoc); | |
| 107 Map<String,Object> storedFields = LuceneUtils.toMap(doc); | |
| 108 log.writeLong(time); | |
| 109 log.writeByte(OP_ADD_DOCUMENT); | |
| 110 log.writeMap(storedFields); | |
| 111 } | |
| 112 }); | |
| 113 reader.close(); | |
| 114 log.commit(); | |
| 115 } | |
| 116 | |
| 117 private LogFile newLogFile() throws IOException { | |
| 118 File file; | |
| 119 do { | |
| 120 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); | |
| 121 } while( file.exists() ); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
122 return LogFile.newLogFile(file); |
| 1461 | 123 } |
| 124 | |
| 1473 | 125 private void deleteUnusedFiles() throws IOException { |
| 1465 | 126 Set<String> used = new HashSet<String>(); |
| 127 used.add( index.getName() ); | |
| 128 for( LogFile lf : logs ) { | |
| 129 used.add( lf.file.getName() ); | |
| 130 } | |
| 131 for( File f : logDir.listFiles() ) { | |
| 132 if( !used.contains(f.getName()) ) { | |
| 1475 | 133 IoUtils.deleteRecursively(f); |
| 1465 | 134 } |
| 135 } | |
| 1461 | 136 } |
| 137 | |
| 1465 | 138 private void writeIndex() throws IOException { |
| 139 ByteArrayOutputStream baos = new ByteArrayOutputStream(); | |
| 140 DataOutputStream dos = new DataOutputStream(baos); | |
| 141 dos.writeInt(version); | |
| 142 dos.writeInt(logs.size()); | |
| 143 for( LogFile lf : logs ) { | |
| 144 String fileName = lf.file.getName(); | |
| 145 dos.writeUTF(fileName); | |
| 146 } | |
| 147 dos.close(); | |
| 148 RandomAccessFile raf = new RandomAccessFile( index, "rwd" ); | |
| 149 raf.write( baos.toByteArray() ); | |
| 150 raf.close(); | |
| 151 deleteUnusedFiles(); | |
| 152 logger.info("writeIndex "+logs.toString()); | |
| 1461 | 153 } |
| 154 | |
| 1465 | 155 private void mergeLogs() throws IOException { |
| 156 logger.info("merge"); | |
| 157 LogFile first = logs.get(0); | |
| 158 LogFile second = logs.get(1); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
159 long lastTime = second.file.lastModified(); |
| 1465 | 160 File dirFile = new File(logDir,"merge"); |
| 161 if( dirFile.exists() ) | |
| 162 throw new RuntimeException(); | |
| 163 Directory dir = FSDirectory.open(dirFile); | |
| 164 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig ); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
165 playLog( first.input(), mergeWriter ); |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
166 playLog( second.input(), mergeWriter ); |
| 1465 | 167 mergeWriter.commit(); |
| 168 LogFile merge = newLogFile(); | |
| 169 logLucene( lastTime, merge, mergeWriter ); | |
| 170 mergeWriter.close(); | |
| 171 synchronized(this) { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
172 //check(); |
| 1465 | 173 logs.remove(0); |
| 174 logs.set(0,merge); | |
| 175 writeIndex(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
176 //check(null); |
| 1465 | 177 } |
| 1461 | 178 } |
| 1465 | 179 private final Runnable mergeLogs = new Runnable() { public void run() { |
| 180 try { | |
| 181 mergeLogs(); | |
| 182 } catch(IOException e) { | |
| 183 throw new RuntimeException(e); | |
| 184 } finally { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
185 isNotMerging(); |
| 1465 | 186 } |
| 187 } }; | |
| 1461 | 188 |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
189 private static class DocIter { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
190 final IndexReader reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
191 final TopDocs td; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
192 final int n; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
193 int i = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
194 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
195 DocIter(IndexReader reader,Query query,Sort sort) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
196 this.reader = reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
197 IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
198 this.td = searcher.search(query,10000000,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
199 this.n = td.scoreDocs.length; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
200 if( td.totalHits != n ) |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
201 throw new RuntimeException(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
202 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
203 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
204 Document next() throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
205 return i < n ? reader.document(td.scoreDocs[i++].doc) : null; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
206 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
207 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
208 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
209 public void check(SortField sortField) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
210 IndexReader indexReader; |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
211 List<LogInputStream> logReaders; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
212 synchronized(this) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
213 if( isMerging ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
214 logger.warn("is merging, check aborted"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
215 return; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
216 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
217 isMerging = true; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
218 indexReader = indexWriter.openReader(); |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
219 logReaders = logReaders(logs); |
| 1465 | 220 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
221 try { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
222 logger.info("check start"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
223 indexWriter.check(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
224 File dirFile = new File(logDir,"check"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
225 IoUtils.deleteRecursively(dirFile); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
226 Directory dir = FSDirectory.open(dirFile); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
227 LuceneIndexWriter checkWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig ); |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
228 playLogs(logReaders,checkWriter); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
229 logger.info("check lucene"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
230 IndexReader checkReader = checkWriter.openReader(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
231 if( sortField == null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
232 int nCheck = checkReader.numDocs(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
233 int nOrig = indexReader.numDocs(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
234 if( nCheck != nOrig ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
235 logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
236 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
237 logger.info("numDocs="+nOrig); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
238 if( hash(indexReader) != hash(checkReader) ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
239 logger.error("hash mismatch"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
240 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
241 } else { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
242 Sort sort = new Sort(sortField); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
243 String sortFieldName = sortField.getField(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
244 Query query = new PrefixQuery(new Term(sortFieldName)); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
245 DocIter origIter = new DocIter(indexReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
246 DocIter checkIter = new DocIter(checkReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
247 Map<String,Object> origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
248 Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
249 while( origFields!=null && checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
250 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
251 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
252 int cmp = origFld.compareTo(checkFld); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
253 if( cmp==0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
254 if( !origFields.equals(checkFields) ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
255 logger.error(sortFieldName+" "+origFld+" not equal"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
256 logger.error("lucene = "+origFields); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
257 logger.error("logs = "+checkFields); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
258 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
259 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
260 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
261 } else if( cmp < 0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
262 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
263 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
264 } else { // > |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
265 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
266 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
267 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
268 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
269 while( origFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
270 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
271 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
272 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
273 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
274 while( checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
275 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
276 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
277 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
278 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
279 //logger.info("check done"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
280 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
281 checkReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
282 checkWriter.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
283 IoUtils.deleteRecursively(dirFile); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
284 logger.info("check done"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
285 } finally { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
286 indexReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
287 isNotMerging(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
288 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
289 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
290 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
291 private static abstract class HashCollector extends GoodCollector { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
292 int total = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
293 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
294 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
295 private static int hash(IndexReader reader) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
296 final IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
297 Query query = new MatchAllDocsQuery(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
298 HashCollector col = new HashCollector() { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
299 public void collectDoc(int iDoc) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
300 Document doc = searcher.doc(iDoc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
301 Map<String,Object> storedFields = LuceneUtils.toMap(doc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
302 total += storedFields.hashCode(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
303 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
304 }; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
305 searcher.search(query,col); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
306 return col.total; |
| 1461 | 307 } |
| 308 | |
| 1465 | 309 private LogFile log() { |
| 310 return logs.get(logs.size()-1); | |
| 311 } | |
| 312 | |
| 313 public synchronized void close() throws IOException { | |
| 314 indexWriter.close(); | |
| 315 LogFile log = log(); | |
| 316 log.commit(); | |
| 317 } | |
| 318 | |
| 319 public synchronized void commit() throws IOException { | |
| 320 indexWriter.commit(); | |
| 321 LogFile log = log(); | |
| 322 log.commit(); | |
| 323 if( isMerging ) | |
| 324 return; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
325 if( log.end() > logs.get(0).end() ) { |
| 1465 | 326 logs.add( newLogFile() ); |
| 327 writeIndex(); | |
| 328 } | |
| 329 if( logs.size() > 3 ) { | |
| 330 isMerging = true; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
331 new Thread(mergeLogs).start(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
332 // mergeLogs.run(); |
| 1465 | 333 } |
| 1461 | 334 } |
| 335 | |
| 1465 | 336 public synchronized void rollback() throws IOException { |
| 337 indexWriter.rollback(); | |
| 338 LogFile log = log(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
339 log.rollback(); |
| 1465 | 340 } |
| 341 | |
| 342 public synchronized void deleteAll() throws IOException { | |
| 343 indexWriter.deleteAll(); | |
| 344 LogFile log = log(); | |
| 345 writeOp(log,OP_DELETE_ALL); | |
| 1461 | 346 } |
| 347 | |
| 1465 | 348 public synchronized void deleteDocuments(Query query) throws IOException { |
| 349 indexWriter.deleteDocuments(query); | |
| 350 LogFile log = log(); | |
| 351 writeOp(log,OP_DELETE_DOCUMENTS); | |
| 352 log.writeQuery(query); | |
| 353 } | |
| 354 | |
| 355 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException { | |
| 356 indexWriter.addDocument(storedFields); | |
| 357 LogFile log = log(); | |
| 358 writeOp(log,OP_ADD_DOCUMENT); | |
| 359 log.writeMap(storedFields); | |
| 360 } | |
| 361 | |
| 362 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { | |
| 363 indexWriter.updateDocument(keyFieldName,storedFields); | |
| 364 LogFile log = log(); | |
| 365 writeOp(log,OP_UPDATE_DOCUMENT); | |
| 366 log.writeUTF(keyFieldName); | |
| 367 log.writeMap(storedFields); | |
| 368 } | |
| 369 | |
| 370 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { | |
| 1461 | 371 indexWriter.reindexDocuments(keyFieldName,query); |
| 372 } | |
| 373 | |
| 1465 | 374 private void writeOp(LogFile log,int op) throws IOException { |
| 375 log.writeLong(System.currentTimeMillis()); | |
| 376 log.writeByte(op); | |
| 377 } | |
| 378 | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
379 public synchronized void playLogs() throws IOException { |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
380 playLogs( logReaders(logs), indexWriter ); |
| 1465 | 381 } |
| 382 | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
383 private static List<LogInputStream> logReaders(List<LogFile> logs) throws IOException { |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
384 List<LogInputStream> logReaders = new ArrayList<LogInputStream>(); |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
385 for( LogFile log : logs ) { |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
386 logReaders.add( log.input() ); |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
387 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
388 return logReaders; |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
389 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
390 |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
391 private static void playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter) throws IOException { |
| 1465 | 392 if( numDocs(indexWriter) != 0 ) |
| 393 throw new RuntimeException ("not empty"); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
394 for( LogInputStream reader : logReaders ) { |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
395 playLog(reader,indexWriter); |
| 1465 | 396 } |
| 397 indexWriter.commit(); | |
| 398 } | |
| 399 | |
| 400 private static int numDocs(LuceneIndexWriter indexWriter) throws IOException { | |
| 401 IndexReader reader = indexWriter.openReader(); | |
| 402 int n = reader.numDocs(); | |
| 403 reader.close(); | |
| 404 return n; | |
| 405 } | |
| 406 | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
407 private static void playLog(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
408 while( in.available() > 0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
409 playOp(in,indexWriter); |
| 1465 | 410 } |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
411 in.close(); |
| 1465 | 412 } |
| 413 | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
414 private static void playOp(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
415 in.readLong(); // time |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
416 int op = in.readByte(); |
| 1461 | 417 switch(op) { |
| 418 case OP_DELETE_ALL: | |
| 419 indexWriter.deleteAll(); | |
| 420 return; | |
| 421 case OP_DELETE_DOCUMENTS: | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
422 indexWriter.deleteDocuments( in.readQuery() ); |
| 1461 | 423 return; |
| 424 case OP_ADD_DOCUMENT: | |
| 1465 | 425 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
426 Map storedFields = in.readMap(); |
| 1465 | 427 indexWriter.addDocument(storedFields); |
| 428 return; | |
| 429 } | |
| 1461 | 430 case OP_UPDATE_DOCUMENT: |
| 1465 | 431 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
432 String keyFieldName = in.readUTF(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
433 Map storedFields = in.readMap(); |
| 1465 | 434 indexWriter.updateDocument(keyFieldName,storedFields); |
| 435 return; | |
| 436 } | |
| 1461 | 437 default: |
| 438 throw new RuntimeException("invalid op "+op); | |
| 439 } | |
| 440 } | |
| 441 | |
| 1465 | 442 private static void dump(LuceneIndexWriter indexWriter) throws IOException { |
| 443 IndexReader reader = indexWriter.openReader(); | |
| 444 IndexSearcher searcher = new IndexSearcher(reader); | |
| 445 Query query = new MatchAllDocsQuery(); | |
| 446 TopDocs td = searcher.search(query,100); | |
| 447 System.out.println("totalHits = "+td.totalHits); | |
| 448 for( int i=0; i<td.scoreDocs.length; i++ ) { | |
| 449 Document doc = searcher.doc(td.scoreDocs[i].doc); | |
| 450 System.out.println(LuceneUtils.toMap(doc)); | |
| 1461 | 451 } |
| 1465 | 452 System.out.println(); |
| 453 reader.close(); | |
| 1461 | 454 } |
| 1465 | 455 |
| 1461 | 456 } |