Mercurial Hosting > luan
annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1775:e380ab6c0c78
minor
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Thu, 27 Jul 2023 17:52:52 -0600 |
parents | 39287902fb0c |
children |
rev | line source |
---|---|
1461 | 1 package goodjava.lucene.logging; |
2 | |
1465 | 3 import java.io.File; |
4 import java.io.RandomAccessFile; | |
5 import java.io.ByteArrayOutputStream; | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
6 import java.io.OutputStreamWriter; |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
7 import java.io.FileReader; |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
8 import java.io.Writer; |
1461 | 9 import java.io.IOException; |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
10 import java.util.List; |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
11 import java.util.ArrayList; |
1461 | 12 import java.util.Map; |
1465 | 13 import java.util.Set; |
14 import java.util.HashSet; | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
15 import java.util.LinkedHashMap; |
1465 | 16 import java.util.Random; |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
17 import java.util.concurrent.TimeUnit; |
1465 | 18 import org.apache.lucene.document.Document; |
19 import org.apache.lucene.index.DirectoryReader; | |
20 import org.apache.lucene.index.IndexReader; | |
1539 | 21 import org.apache.lucene.index.IndexWriter; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
22 import org.apache.lucene.index.Term; |
1465 | 23 import org.apache.lucene.search.IndexSearcher; |
1461 | 24 import org.apache.lucene.search.Query; |
1465 | 25 import org.apache.lucene.search.MatchAllDocsQuery; |
26 import org.apache.lucene.search.TopDocs; | |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
27 import org.apache.lucene.search.PrefixQuery; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
28 import org.apache.lucene.search.SortField; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
29 import org.apache.lucene.search.Sort; |
1465 | 30 import org.apache.lucene.store.Directory; |
31 import org.apache.lucene.store.FSDirectory; | |
1473 | 32 import goodjava.io.IoUtils; |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
33 import goodjava.json.JsonParser; |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
34 import goodjava.json.JsonToString; |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
35 import goodjava.parser.ParseException; |
1461 | 36 import goodjava.lucene.api.GoodIndexWriter; |
1465 | 37 import goodjava.lucene.api.LuceneIndexWriter; |
38 import goodjava.lucene.api.GoodCollector; | |
39 import goodjava.lucene.api.LuceneUtils; | |
40 import goodjava.logging.Logger; | |
41 import goodjava.logging.LoggerFactory; | |
1461 | 42 |
43 | |
1488 | 44 public class LoggingIndexWriter implements GoodIndexWriter { |
1465 | 45 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class); |
1548 | 46 private static final int version = 2; |
1461 | 47 private static final int OP_DELETE_ALL = 1; |
48 private static final int OP_DELETE_DOCUMENTS = 2; | |
49 private static final int OP_ADD_DOCUMENT = 3; | |
50 private static final int OP_UPDATE_DOCUMENT = 4; | |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
51 private static final int OP_TAG = 5; |
1465 | 52 private static final Random rnd = new Random(); |
1461 | 53 |
1465 | 54 public final LuceneIndexWriter indexWriter; |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
55 public boolean wasCreated; |
1465 | 56 private final File logDir; |
1548 | 57 private final long logTime; |
58 protected final LogFile[] logs = new LogFile[3]; | |
1486 | 59 private LogOutputStream log; |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
60 private final File indexFile; |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
61 private final SemaphoreLock mergeLock = new SemaphoreLock(); |
1461 | 62 |
1548 | 63 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime) |
64 throws IOException | |
65 { | |
1461 | 66 this.indexWriter = indexWriter; |
1465 | 67 this.logDir = logDir; |
1548 | 68 this.logTime = logTime; |
1501 | 69 IoUtils.mkdirs(logDir); |
1465 | 70 if( !logDir.isDirectory() ) |
71 throw new RuntimeException(); | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
72 indexFile = new File(logDir,"index.json"); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
73 if( indexFile.exists() ) { |
1465 | 74 try { |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
75 Map map = (Map)JsonParser.parse( IoUtils.readAll(new FileReader(indexFile)) ); |
1679 | 76 if( ((Number)map.get("version")).intValue() == version ) { |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
77 List fileNames = (List)map.get("files"); |
1548 | 78 for( int i=0; i<logs.length; i++ ) { |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
79 File file = new File( logDir, (String)fileNames.get(i) ); |
1548 | 80 logs[i] = new LogFile(file); |
1465 | 81 } |
82 deleteUnusedFiles(); | |
1486 | 83 setLog(); |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
84 wasCreated = false; |
1465 | 85 return; |
86 } | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
87 } catch(ParseException e) { |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
88 logger.error("bad index.json",e); |
1465 | 89 } |
90 } | |
1556 | 91 logger.info("building new logs"); |
92 for( int i=0; i<logs.length; i++ ) { | |
93 logs[i] = newLogFile(); | |
94 } | |
95 LogOutputStream log = logs[0].output(); | |
96 logLucene( System.currentTimeMillis(), log, indexWriter ); | |
97 log.close(); | |
98 writeIndex(); | |
99 setLog(); | |
100 logger.info("done building new logs"); | |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
101 wasCreated = true; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
102 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
103 |
1548 | 104 public IndexReader openReader() throws IOException { |
105 return indexWriter.openReader(); | |
106 } | |
107 | |
1539 | 108 public IndexWriter getLuceneIndexWriter() { |
109 return indexWriter.getLuceneIndexWriter(); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
110 } |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
111 |
1486 | 112 private void setLog() throws IOException { |
113 if( log != null ) | |
114 log.close(); | |
1548 | 115 log = logs[2].output(); |
1486 | 116 } |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
117 /* |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
118 public synchronized boolean isMerging() { |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
119 return mergeLock.isLocked(); |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
120 } |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
121 */ |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
122 private void getMergeLock() { |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
123 try { |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
124 if( !mergeLock.tryLock(1,TimeUnit.MINUTES) ) |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
125 throw new RuntimeException("failed to acquire lock"); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
126 } catch(InterruptedException e) { |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
127 throw new RuntimeException(e); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
128 } |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
129 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
130 |
1548 | 131 public synchronized void logLucene() |
132 throws IOException | |
133 { | |
134 logLucene( System.currentTimeMillis(), log, indexWriter ); | |
135 } | |
136 | |
137 private static void logLucene(long time,LogOutputStream log,LuceneIndexWriter indexWriter) | |
138 throws IOException | |
139 { | |
140 log.writeLong(time); | |
141 log.writeByte(OP_DELETE_ALL); | |
1465 | 142 IndexReader reader = indexWriter.openReader(); |
143 final IndexSearcher searcher = new IndexSearcher(reader); | |
144 Query query = new MatchAllDocsQuery(); | |
145 searcher.search( query, new GoodCollector(){ | |
146 public void collectDoc(int iDoc) throws IOException { | |
147 Document doc = searcher.doc(iDoc); | |
148 Map<String,Object> storedFields = LuceneUtils.toMap(doc); | |
149 log.writeLong(time); | |
150 log.writeByte(OP_ADD_DOCUMENT); | |
151 log.writeMap(storedFields); | |
152 } | |
153 }); | |
154 reader.close(); | |
155 log.commit(); | |
156 } | |
157 | |
158 private LogFile newLogFile() throws IOException { | |
159 File file; | |
160 do { | |
161 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); | |
162 } while( file.exists() ); | |
1486 | 163 return new LogFile(file); |
1461 | 164 } |
165 | |
1473 | 166 private void deleteUnusedFiles() throws IOException { |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
167 deleteUnusedFiles(logs,indexFile); |
1499 | 168 } |
169 | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
170 private static void deleteUnusedFiles(LogFile[] logs,File indexFile) throws IOException { |
1465 | 171 Set<String> used = new HashSet<String>(); |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
172 used.add( indexFile.getName() ); |
1465 | 173 for( LogFile lf : logs ) { |
174 used.add( lf.file.getName() ); | |
175 } | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
176 for( File f : indexFile.getParentFile().listFiles() ) { |
1465 | 177 if( !used.contains(f.getName()) ) { |
1475 | 178 IoUtils.deleteRecursively(f); |
1465 | 179 } |
180 } | |
1461 | 181 } |
182 | |
1465 | 183 private void writeIndex() throws IOException { |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
184 writeIndex(logs,indexFile); |
1499 | 185 } |
186 | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
187 public static void writeIndex(LogFile[] logs,File indexFile) throws IOException { |
1548 | 188 if( logs.length != 3 ) |
189 throw new RuntimeException(); | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
190 Map map = new LinkedHashMap(); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
191 map.put("version",version); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
192 List fileNames = new ArrayList(); |
1465 | 193 for( LogFile lf : logs ) { |
194 String fileName = lf.file.getName(); | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
195 fileNames.add(fileName); |
1465 | 196 } |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
197 map.put("files",fileNames); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
198 ByteArrayOutputStream baos = new ByteArrayOutputStream(); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
199 Writer writer = new OutputStreamWriter(baos); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
200 writer.write( new JsonToString().toString(map) ); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
201 writer.write( '\n' ); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
202 writer.close(); |
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
203 RandomAccessFile raf = new RandomAccessFile( indexFile, "rwd" ); |
1465 | 204 raf.write( baos.toByteArray() ); |
205 raf.close(); | |
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
206 deleteUnusedFiles(logs,indexFile); |
1499 | 207 //logger.info("writeIndex "+logs.toString()); |
1461 | 208 } |
209 | |
1465 | 210 private void mergeLogs() throws IOException { |
1548 | 211 logger.info("merge"); |
212 if( !mergeLock.isLocked() ) { | |
213 logger.error("merge without lock"); | |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
214 return; |
1548 | 215 } |
216 LogFile first = logs[0]; | |
217 LogFile second = logs[1]; | |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
218 long lastTime = second.file.lastModified(); |
1465 | 219 File dirFile = new File(logDir,"merge"); |
220 if( dirFile.exists() ) | |
221 throw new RuntimeException(); | |
222 Directory dir = FSDirectory.open(dirFile); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
223 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
1551 | 224 OpDoer opDoer = new BasicOpDoer(mergeWriter); |
1549 | 225 playLog( first.input(), opDoer ); |
226 playLog( second.input(), opDoer ); | |
1465 | 227 mergeWriter.commit(); |
228 LogFile merge = newLogFile(); | |
1548 | 229 LogOutputStream log = merge.output(); |
230 logLucene( lastTime, log, mergeWriter ); | |
231 log.close(); | |
1465 | 232 mergeWriter.close(); |
233 synchronized(this) { | |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
234 //check(); |
1548 | 235 logs[0] = merge; |
236 logs[1] = logs[2]; | |
237 logs[2] = newLogFile(); | |
1465 | 238 writeIndex(); |
1548 | 239 setLog(); |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
240 //check(null); |
1465 | 241 } |
1461 | 242 } |
1465 | 243 private final Runnable mergeLogs = new Runnable() { public void run() { |
244 try { | |
245 mergeLogs(); | |
246 } catch(IOException e) { | |
247 throw new RuntimeException(e); | |
248 } finally { | |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
249 mergeLock.unlock(); |
1465 | 250 } |
251 } }; | |
1461 | 252 |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
253 private static class DocIter { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
254 final IndexReader reader; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
255 final TopDocs td; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
256 final int n; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
257 int i = 0; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
258 |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
259 DocIter(IndexReader reader,Query query,Sort sort) throws IOException { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
260 this.reader = reader; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
261 IndexSearcher searcher = new IndexSearcher(reader); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
262 this.td = searcher.search(query,10000000,sort); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
263 this.n = td.scoreDocs.length; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
264 if( td.totalHits != n ) |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
265 throw new RuntimeException(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
266 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
267 |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
268 Document next() throws IOException { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
269 return i < n ? reader.document(td.scoreDocs[i++].doc) : null; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
270 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
271 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
272 |
1487 | 273 private volatile boolean isChecking = false; |
274 | |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
275 public boolean check(SortField sortField) throws IOException { |
1487 | 276 if( isChecking ) |
277 throw new RuntimeException("another check is running"); | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
278 isChecking = true; |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
279 try { |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
280 return doCheck(sortField); |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
281 } finally { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
282 isChecking = false; |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
283 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
284 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
285 |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
286 protected boolean doCheck(SortField sortField) throws IOException { |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
287 boolean ok = true; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
288 IndexReader indexReader; |
1548 | 289 LogInputStream[] logReaders; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
290 synchronized(this) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
291 indexReader = indexWriter.openReader(); |
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
292 logReaders = logReaders(logs); |
1465 | 293 } |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
294 try { |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
295 //logger.info("check start"); |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
296 indexWriter.check(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
297 File dirFile = new File(logDir,"check"); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
298 IoUtils.deleteRecursively(dirFile); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
299 Directory dir = FSDirectory.open(dirFile); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
300 LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
1551 | 301 playLogs(logReaders,new BasicOpDoer(checkWriter)); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
302 //logger.info("check lucene"); |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
303 IndexReader checkReader = checkWriter.openReader(); |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
304 int nCheck = checkReader.numDocs(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
305 int nOrig = indexReader.numDocs(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
306 if( nCheck != nOrig ) { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
307 logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
308 ok = false; |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
309 } |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
310 if( sortField == null ) { |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
311 if( ok && hash(indexReader) != hash(checkReader) ) { |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
312 logger.error("hash mismatch"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
313 ok = false; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
314 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
315 } else { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
316 Sort sort = new Sort(sortField); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
317 String sortFieldName = sortField.getField(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
318 Query query = new PrefixQuery(new Term(sortFieldName)); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
319 DocIter origIter = new DocIter(indexReader,query,sort); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
320 DocIter checkIter = new DocIter(checkReader,query,sort); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
321 Map<String,Object> origFields = LuceneUtils.toMap(origIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
322 Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
323 while( origFields!=null && checkFields!=null ) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
324 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
325 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
326 int cmp = origFld.compareTo(checkFld); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
327 if( cmp==0 ) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
328 if( !origFields.equals(checkFields) ) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
329 logger.error(sortFieldName+" "+origFld+" not equal"); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
330 logger.error("lucene = "+origFields); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
331 logger.error("logs = "+checkFields); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
332 ok = false; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
333 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
334 origFields = LuceneUtils.toMap(origIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
335 checkFields = LuceneUtils.toMap(checkIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
336 } else if( cmp < 0 ) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
337 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
338 ok = false; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
339 origFields = LuceneUtils.toMap(origIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
340 } else { // > |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
341 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
342 ok = false; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
343 checkFields = LuceneUtils.toMap(checkIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
344 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
345 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
346 while( origFields!=null ) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
347 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
348 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
349 ok = false; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
350 origFields = LuceneUtils.toMap(origIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
351 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
352 while( checkFields!=null ) { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
353 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
354 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
355 ok = false; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
356 checkFields = LuceneUtils.toMap(checkIter.next()); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
357 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
358 //logger.info("check done"); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
359 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
360 checkReader.close(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
361 checkWriter.close(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
362 IoUtils.deleteRecursively(dirFile); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
363 //logger.info("check done"); |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
364 } finally { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
365 indexReader.close(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
366 } |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
367 return ok; |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
368 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
369 |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
370 private static abstract class HashCollector extends GoodCollector { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
371 int total = 0; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
372 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
373 |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
374 private static int hash(IndexReader reader) throws IOException { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
375 final IndexSearcher searcher = new IndexSearcher(reader); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
376 Query query = new MatchAllDocsQuery(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
377 HashCollector col = new HashCollector() { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
378 public void collectDoc(int iDoc) throws IOException { |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
379 Document doc = searcher.doc(iDoc); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
380 Map<String,Object> storedFields = LuceneUtils.toMap(doc); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
381 total += storedFields.hashCode(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
382 } |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
383 }; |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
384 searcher.search(query,col); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
385 return col.total; |
1461 | 386 } |
387 | |
1465 | 388 public synchronized void close() throws IOException { |
389 indexWriter.close(); | |
390 log.commit(); | |
1486 | 391 log.close(); |
1465 | 392 } |
393 | |
394 public synchronized void commit() throws IOException { | |
395 indexWriter.commit(); | |
396 log.commit(); | |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
397 if( mergeLock.isLocked() ) |
1465 | 398 return; |
1548 | 399 if( logs[1].file.lastModified() < System.currentTimeMillis() - logTime ) { |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
400 getMergeLock(); |
1504 | 401 new Thread(mergeLogs).start(); |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
402 // mergeLogs.run(); |
1465 | 403 } |
1461 | 404 } |
405 | |
1465 | 406 public synchronized void rollback() throws IOException { |
407 indexWriter.rollback(); | |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
408 log.rollback(); |
1465 | 409 } |
410 | |
411 public synchronized void deleteAll() throws IOException { | |
412 indexWriter.deleteAll(); | |
1486 | 413 writeOp(OP_DELETE_ALL); |
1461 | 414 } |
415 | |
1465 | 416 public synchronized void deleteDocuments(Query query) throws IOException { |
417 indexWriter.deleteDocuments(query); | |
1486 | 418 writeOp(OP_DELETE_DOCUMENTS); |
1465 | 419 log.writeQuery(query); |
420 } | |
421 | |
422 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException { | |
423 indexWriter.addDocument(storedFields); | |
1486 | 424 writeOp(OP_ADD_DOCUMENT); |
1465 | 425 log.writeMap(storedFields); |
426 } | |
427 | |
428 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { | |
429 indexWriter.updateDocument(keyFieldName,storedFields); | |
1486 | 430 writeOp(OP_UPDATE_DOCUMENT); |
1465 | 431 log.writeUTF(keyFieldName); |
432 log.writeMap(storedFields); | |
433 } | |
434 | |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
435 public synchronized void tag(String tag) throws IOException { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
436 writeOp(OP_TAG); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
437 log.writeUTF(tag); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
438 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
439 |
1465 | 440 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { |
1461 | 441 indexWriter.reindexDocuments(keyFieldName,query); |
442 } | |
443 | |
1486 | 444 private void writeOp(int op) throws IOException { |
1465 | 445 log.writeLong(System.currentTimeMillis()); |
446 log.writeByte(op); | |
447 } | |
448 | |
1549 | 449 public synchronized void playLogs(OpDoer opDoer) throws IOException { |
450 if( opDoer == null ) | |
1551 | 451 opDoer = new BasicOpDoer(indexWriter); |
1549 | 452 playLogs( logReaders(logs), opDoer ); |
1465 | 453 } |
454 | |
1548 | 455 private static LogInputStream[] logReaders(LogFile[] logs) throws IOException { |
456 LogInputStream[] logReaders = new LogInputStream[logs.length]; | |
457 for( int i=0; i<logs.length; i++ ) { | |
458 logReaders[i] = logs[i].input(); | |
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
459 } |
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
460 return logReaders; |
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
461 } |
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
462 |
1549 | 463 private static void playLogs(LogInputStream[] logReaders,OpDoer opDoer) |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
464 throws IOException |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
465 { |
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
466 for( LogInputStream reader : logReaders ) { |
1549 | 467 playLog(reader,opDoer); |
1465 | 468 } |
1549 | 469 opDoer.commit(); |
1465 | 470 } |
471 | |
1549 | 472 private static void playLog(LogInputStream in,OpDoer opDoer) |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
473 throws IOException |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
474 { |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
475 while( in.available() > 0 ) { |
1549 | 476 playOp(in,opDoer); |
1465 | 477 } |
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
478 in.close(); |
1465 | 479 } |
480 | |
1549 | 481 private static void playOp(LogInputStream in,OpDoer opDoer) |
1548 | 482 throws IOException |
483 { | |
1549 | 484 long time = in.readLong(); // time |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
485 int op = in.readByte(); |
1461 | 486 switch(op) { |
487 case OP_DELETE_ALL: | |
1549 | 488 opDoer.deleteAll(time); |
1548 | 489 return; |
1461 | 490 case OP_DELETE_DOCUMENTS: |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
491 { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
492 Query query = in.readQuery(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
493 //System.out.println("OP_DELETE_DOCUMENTS "+query); |
1549 | 494 opDoer.deleteDocuments(time,query); |
1548 | 495 return; |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
496 } |
1461 | 497 case OP_ADD_DOCUMENT: |
1465 | 498 { |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
499 Map storedFields = in.readMap(); |
1549 | 500 opDoer.addDocument(time,storedFields); |
1548 | 501 return; |
1465 | 502 } |
1461 | 503 case OP_UPDATE_DOCUMENT: |
1465 | 504 { |
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
505 String keyFieldName = in.readUTF(); |
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
506 Map storedFields = in.readMap(); |
1549 | 507 opDoer.updateDocument(time,keyFieldName,storedFields); |
1548 | 508 return; |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
509 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
510 case OP_TAG: |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
511 { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
512 String tag = in.readUTF(); |
1549 | 513 opDoer.tag(time,tag); |
1548 | 514 return; |
1465 | 515 } |
1461 | 516 default: |
517 throw new RuntimeException("invalid op "+op); | |
518 } | |
519 } | |
520 | |
1465 | 521 private static void dump(LuceneIndexWriter indexWriter) throws IOException { |
522 IndexReader reader = indexWriter.openReader(); | |
523 IndexSearcher searcher = new IndexSearcher(reader); | |
524 Query query = new MatchAllDocsQuery(); | |
525 TopDocs td = searcher.search(query,100); | |
526 System.out.println("totalHits = "+td.totalHits); | |
527 for( int i=0; i<td.scoreDocs.length; i++ ) { | |
528 Document doc = searcher.doc(td.scoreDocs[i].doc); | |
529 System.out.println(LuceneUtils.toMap(doc)); | |
1461 | 530 } |
1465 | 531 System.out.println(); |
532 reader.close(); | |
1461 | 533 } |
1465 | 534 |
1461 | 535 } |