Mercurial Hosting > luan
annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1678:f54484aae295
fix
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Tue, 31 May 2022 14:44:29 -0600 |
| parents | ea7075b7afe1 |
| children | 39287902fb0c |
| rev | line source |
|---|---|
| 1461 | 1 package goodjava.lucene.logging; |
| 2 | |
| 1465 | 3 import java.io.File; |
| 4 import java.io.RandomAccessFile; | |
| 5 import java.io.ByteArrayOutputStream; | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
6 import java.io.OutputStreamWriter; |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
7 import java.io.FileReader; |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
8 import java.io.Writer; |
| 1461 | 9 import java.io.IOException; |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
10 import java.util.List; |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
11 import java.util.ArrayList; |
| 1461 | 12 import java.util.Map; |
| 1465 | 13 import java.util.Set; |
| 14 import java.util.HashSet; | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
15 import java.util.LinkedHashMap; |
| 1465 | 16 import java.util.Random; |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
17 import java.util.concurrent.TimeUnit; |
| 1465 | 18 import org.apache.lucene.document.Document; |
| 19 import org.apache.lucene.index.DirectoryReader; | |
| 20 import org.apache.lucene.index.IndexReader; | |
| 1539 | 21 import org.apache.lucene.index.IndexWriter; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
22 import org.apache.lucene.index.Term; |
| 1465 | 23 import org.apache.lucene.search.IndexSearcher; |
| 1461 | 24 import org.apache.lucene.search.Query; |
| 1465 | 25 import org.apache.lucene.search.MatchAllDocsQuery; |
| 26 import org.apache.lucene.search.TopDocs; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
27 import org.apache.lucene.search.PrefixQuery; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
28 import org.apache.lucene.search.SortField; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
29 import org.apache.lucene.search.Sort; |
| 1465 | 30 import org.apache.lucene.store.Directory; |
| 31 import org.apache.lucene.store.FSDirectory; | |
| 1473 | 32 import goodjava.io.IoUtils; |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
33 import goodjava.json.JsonParser; |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
34 import goodjava.json.JsonToString; |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
35 import goodjava.parser.ParseException; |
| 1461 | 36 import goodjava.lucene.api.GoodIndexWriter; |
| 1465 | 37 import goodjava.lucene.api.LuceneIndexWriter; |
| 38 import goodjava.lucene.api.GoodCollector; | |
| 39 import goodjava.lucene.api.LuceneUtils; | |
| 40 import goodjava.logging.Logger; | |
| 41 import goodjava.logging.LoggerFactory; | |
| 1461 | 42 |
| 43 | |
| 1488 | 44 public class LoggingIndexWriter implements GoodIndexWriter { |
| 1465 | 45 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class); |
| 1548 | 46 private static final int version = 2; |
| 1461 | 47 private static final int OP_DELETE_ALL = 1; |
| 48 private static final int OP_DELETE_DOCUMENTS = 2; | |
| 49 private static final int OP_ADD_DOCUMENT = 3; | |
| 50 private static final int OP_UPDATE_DOCUMENT = 4; | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
51 private static final int OP_TAG = 5; |
| 1465 | 52 private static final Random rnd = new Random(); |
| 1461 | 53 |
| 1465 | 54 public final LuceneIndexWriter indexWriter; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
55 public boolean wasCreated; |
| 1465 | 56 private final File logDir; |
| 1548 | 57 private final long logTime; |
| 58 protected final LogFile[] logs = new LogFile[3]; | |
| 1486 | 59 private LogOutputStream log; |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
60 private final File indexFile; |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
61 private final SemaphoreLock mergeLock = new SemaphoreLock(); |
| 1461 | 62 |
| 1548 | 63 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime) |
| 64 throws IOException | |
| 65 { | |
| 1461 | 66 this.indexWriter = indexWriter; |
| 1465 | 67 this.logDir = logDir; |
| 1548 | 68 this.logTime = logTime; |
| 1501 | 69 IoUtils.mkdirs(logDir); |
| 1465 | 70 if( !logDir.isDirectory() ) |
| 71 throw new RuntimeException(); | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
72 indexFile = new File(logDir,"index.json"); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
73 if( indexFile.exists() ) { |
| 1465 | 74 try { |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
75 Map map = (Map)JsonParser.parse( IoUtils.readAll(new FileReader(indexFile)) ); |
| 1678 | 76 Number num = (Number)map.get("version"); |
| 77 if( num != null && num.intValue() == version ) { | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
78 List fileNames = (List)map.get("files"); |
| 1548 | 79 for( int i=0; i<logs.length; i++ ) { |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
80 File file = new File( logDir, (String)fileNames.get(i) ); |
| 1548 | 81 logs[i] = new LogFile(file); |
| 1465 | 82 } |
| 83 deleteUnusedFiles(); | |
| 1486 | 84 setLog(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
85 wasCreated = false; |
| 1465 | 86 return; |
| 87 } | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
88 } catch(ParseException e) { |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
89 logger.error("bad index.json",e); |
| 1465 | 90 } |
| 91 } | |
| 1556 | 92 logger.info("building new logs"); |
| 93 for( int i=0; i<logs.length; i++ ) { | |
| 94 logs[i] = newLogFile(); | |
| 95 } | |
| 96 LogOutputStream log = logs[0].output(); | |
| 97 logLucene( System.currentTimeMillis(), log, indexWriter ); | |
| 98 log.close(); | |
| 99 writeIndex(); | |
| 100 setLog(); | |
| 101 logger.info("done building new logs"); | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
102 wasCreated = true; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
103 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
104 |
| 1548 | 105 public IndexReader openReader() throws IOException { |
| 106 return indexWriter.openReader(); | |
| 107 } | |
| 108 | |
| 1539 | 109 public IndexWriter getLuceneIndexWriter() { |
| 110 return indexWriter.getLuceneIndexWriter(); | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
111 } |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
112 |
| 1486 | 113 private void setLog() throws IOException { |
| 114 if( log != null ) | |
| 115 log.close(); | |
| 1548 | 116 log = logs[2].output(); |
| 1486 | 117 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
118 /* |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
119 public synchronized boolean isMerging() { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
120 return mergeLock.isLocked(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
121 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
122 */ |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
123 private void getMergeLock() { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
124 try { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
125 if( !mergeLock.tryLock(1,TimeUnit.MINUTES) ) |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
126 throw new RuntimeException("failed to acquire lock"); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
127 } catch(InterruptedException e) { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
128 throw new RuntimeException(e); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
129 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
130 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
131 |
| 1548 | 132 public synchronized void logLucene() |
| 133 throws IOException | |
| 134 { | |
| 135 logLucene( System.currentTimeMillis(), log, indexWriter ); | |
| 136 } | |
| 137 | |
| 138 private static void logLucene(long time,LogOutputStream log,LuceneIndexWriter indexWriter) | |
| 139 throws IOException | |
| 140 { | |
| 141 log.writeLong(time); | |
| 142 log.writeByte(OP_DELETE_ALL); | |
| 1465 | 143 IndexReader reader = indexWriter.openReader(); |
| 144 final IndexSearcher searcher = new IndexSearcher(reader); | |
| 145 Query query = new MatchAllDocsQuery(); | |
| 146 searcher.search( query, new GoodCollector(){ | |
| 147 public void collectDoc(int iDoc) throws IOException { | |
| 148 Document doc = searcher.doc(iDoc); | |
| 149 Map<String,Object> storedFields = LuceneUtils.toMap(doc); | |
| 150 log.writeLong(time); | |
| 151 log.writeByte(OP_ADD_DOCUMENT); | |
| 152 log.writeMap(storedFields); | |
| 153 } | |
| 154 }); | |
| 155 reader.close(); | |
| 156 log.commit(); | |
| 157 } | |
| 158 | |
| 159 private LogFile newLogFile() throws IOException { | |
| 160 File file; | |
| 161 do { | |
| 162 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); | |
| 163 } while( file.exists() ); | |
| 1486 | 164 return new LogFile(file); |
| 1461 | 165 } |
| 166 | |
| 1473 | 167 private void deleteUnusedFiles() throws IOException { |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
168 deleteUnusedFiles(logs,indexFile); |
| 1499 | 169 } |
| 170 | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
171 private static void deleteUnusedFiles(LogFile[] logs,File indexFile) throws IOException { |
| 1465 | 172 Set<String> used = new HashSet<String>(); |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
173 used.add( indexFile.getName() ); |
| 1465 | 174 for( LogFile lf : logs ) { |
| 175 used.add( lf.file.getName() ); | |
| 176 } | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
177 for( File f : indexFile.getParentFile().listFiles() ) { |
| 1465 | 178 if( !used.contains(f.getName()) ) { |
| 1475 | 179 IoUtils.deleteRecursively(f); |
| 1465 | 180 } |
| 181 } | |
| 1461 | 182 } |
| 183 | |
| 1465 | 184 private void writeIndex() throws IOException { |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
185 writeIndex(logs,indexFile); |
| 1499 | 186 } |
| 187 | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
188 public static void writeIndex(LogFile[] logs,File indexFile) throws IOException { |
| 1548 | 189 if( logs.length != 3 ) |
| 190 throw new RuntimeException(); | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
191 Map map = new LinkedHashMap(); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
192 map.put("version",version); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
193 List fileNames = new ArrayList(); |
| 1465 | 194 for( LogFile lf : logs ) { |
| 195 String fileName = lf.file.getName(); | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
196 fileNames.add(fileName); |
| 1465 | 197 } |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
198 map.put("files",fileNames); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
199 ByteArrayOutputStream baos = new ByteArrayOutputStream(); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
200 Writer writer = new OutputStreamWriter(baos); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
201 writer.write( new JsonToString().toString(map) ); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
202 writer.write( '\n' ); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
203 writer.close(); |
|
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
204 RandomAccessFile raf = new RandomAccessFile( indexFile, "rwd" ); |
| 1465 | 205 raf.write( baos.toByteArray() ); |
| 206 raf.close(); | |
|
1677
ea7075b7afe1
switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents:
1556
diff
changeset
|
207 deleteUnusedFiles(logs,indexFile); |
| 1499 | 208 //logger.info("writeIndex "+logs.toString()); |
| 1461 | 209 } |
| 210 | |
| 1465 | 211 private void mergeLogs() throws IOException { |
| 1548 | 212 logger.info("merge"); |
| 213 if( !mergeLock.isLocked() ) { | |
| 214 logger.error("merge without lock"); | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
215 return; |
| 1548 | 216 } |
| 217 LogFile first = logs[0]; | |
| 218 LogFile second = logs[1]; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
219 long lastTime = second.file.lastModified(); |
| 1465 | 220 File dirFile = new File(logDir,"merge"); |
| 221 if( dirFile.exists() ) | |
| 222 throw new RuntimeException(); | |
| 223 Directory dir = FSDirectory.open(dirFile); | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
224 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
| 1551 | 225 OpDoer opDoer = new BasicOpDoer(mergeWriter); |
| 1549 | 226 playLog( first.input(), opDoer ); |
| 227 playLog( second.input(), opDoer ); | |
| 1465 | 228 mergeWriter.commit(); |
| 229 LogFile merge = newLogFile(); | |
| 1548 | 230 LogOutputStream log = merge.output(); |
| 231 logLucene( lastTime, log, mergeWriter ); | |
| 232 log.close(); | |
| 1465 | 233 mergeWriter.close(); |
| 234 synchronized(this) { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
235 //check(); |
| 1548 | 236 logs[0] = merge; |
| 237 logs[1] = logs[2]; | |
| 238 logs[2] = newLogFile(); | |
| 1465 | 239 writeIndex(); |
| 1548 | 240 setLog(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
241 //check(null); |
| 1465 | 242 } |
| 1461 | 243 } |
| 1465 | 244 private final Runnable mergeLogs = new Runnable() { public void run() { |
| 245 try { | |
| 246 mergeLogs(); | |
| 247 } catch(IOException e) { | |
| 248 throw new RuntimeException(e); | |
| 249 } finally { | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
250 mergeLock.unlock(); |
| 1465 | 251 } |
| 252 } }; | |
| 1461 | 253 |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
254 private static class DocIter { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
255 final IndexReader reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
256 final TopDocs td; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
257 final int n; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
258 int i = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
259 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
260 DocIter(IndexReader reader,Query query,Sort sort) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
261 this.reader = reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
262 IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
263 this.td = searcher.search(query,10000000,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
264 this.n = td.scoreDocs.length; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
265 if( td.totalHits != n ) |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
266 throw new RuntimeException(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
267 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
268 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
269 Document next() throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
270 return i < n ? reader.document(td.scoreDocs[i++].doc) : null; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
271 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
272 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
273 |
| 1487 | 274 private volatile boolean isChecking = false; |
| 275 | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
276 public boolean check(SortField sortField) throws IOException { |
| 1487 | 277 if( isChecking ) |
| 278 throw new RuntimeException("another check is running"); | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
279 isChecking = true; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
280 try { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
281 return doCheck(sortField); |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
282 } finally { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
283 isChecking = false; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
284 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
285 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
286 |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
287 protected boolean doCheck(SortField sortField) throws IOException { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
288 boolean ok = true; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
289 IndexReader indexReader; |
| 1548 | 290 LogInputStream[] logReaders; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
291 synchronized(this) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
292 indexReader = indexWriter.openReader(); |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
293 logReaders = logReaders(logs); |
| 1465 | 294 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
295 try { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
296 //logger.info("check start"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
297 indexWriter.check(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
298 File dirFile = new File(logDir,"check"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
299 IoUtils.deleteRecursively(dirFile); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
300 Directory dir = FSDirectory.open(dirFile); |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
301 LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
| 1551 | 302 playLogs(logReaders,new BasicOpDoer(checkWriter)); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
303 //logger.info("check lucene"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
304 IndexReader checkReader = checkWriter.openReader(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
305 int nCheck = checkReader.numDocs(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
306 int nOrig = indexReader.numDocs(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
307 if( nCheck != nOrig ) { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
308 logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
309 ok = false; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
310 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
311 if( sortField == null ) { |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
312 if( ok && hash(indexReader) != hash(checkReader) ) { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
313 logger.error("hash mismatch"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
314 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
315 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
316 } else { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
317 Sort sort = new Sort(sortField); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
318 String sortFieldName = sortField.getField(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
319 Query query = new PrefixQuery(new Term(sortFieldName)); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
320 DocIter origIter = new DocIter(indexReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
321 DocIter checkIter = new DocIter(checkReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
322 Map<String,Object> origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
323 Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
324 while( origFields!=null && checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
325 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
326 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
327 int cmp = origFld.compareTo(checkFld); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
328 if( cmp==0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
329 if( !origFields.equals(checkFields) ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
330 logger.error(sortFieldName+" "+origFld+" not equal"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
331 logger.error("lucene = "+origFields); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
332 logger.error("logs = "+checkFields); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
333 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
334 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
335 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
336 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
337 } else if( cmp < 0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
338 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
339 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
340 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
341 } else { // > |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
342 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
343 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
344 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
345 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
346 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
347 while( origFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
348 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
349 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
350 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
351 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
352 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
353 while( checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
354 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
355 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
356 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
357 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
358 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
359 //logger.info("check done"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
360 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
361 checkReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
362 checkWriter.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
363 IoUtils.deleteRecursively(dirFile); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
364 //logger.info("check done"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
365 } finally { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
366 indexReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
367 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
368 return ok; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
369 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
370 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
371 private static abstract class HashCollector extends GoodCollector { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
372 int total = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
373 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
374 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
375 private static int hash(IndexReader reader) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
376 final IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
377 Query query = new MatchAllDocsQuery(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
378 HashCollector col = new HashCollector() { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
379 public void collectDoc(int iDoc) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
380 Document doc = searcher.doc(iDoc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
381 Map<String,Object> storedFields = LuceneUtils.toMap(doc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
382 total += storedFields.hashCode(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
383 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
384 }; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
385 searcher.search(query,col); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
386 return col.total; |
| 1461 | 387 } |
| 388 | |
| 1465 | 389 public synchronized void close() throws IOException { |
| 390 indexWriter.close(); | |
| 391 log.commit(); | |
| 1486 | 392 log.close(); |
| 1465 | 393 } |
| 394 | |
| 395 public synchronized void commit() throws IOException { | |
| 396 indexWriter.commit(); | |
| 397 log.commit(); | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
398 if( mergeLock.isLocked() ) |
| 1465 | 399 return; |
| 1548 | 400 if( logs[1].file.lastModified() < System.currentTimeMillis() - logTime ) { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
401 getMergeLock(); |
| 1504 | 402 new Thread(mergeLogs).start(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
403 // mergeLogs.run(); |
| 1465 | 404 } |
| 1461 | 405 } |
| 406 | |
| 1465 | 407 public synchronized void rollback() throws IOException { |
| 408 indexWriter.rollback(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
409 log.rollback(); |
| 1465 | 410 } |
| 411 | |
| 412 public synchronized void deleteAll() throws IOException { | |
| 413 indexWriter.deleteAll(); | |
| 1486 | 414 writeOp(OP_DELETE_ALL); |
| 1461 | 415 } |
| 416 | |
| 1465 | 417 public synchronized void deleteDocuments(Query query) throws IOException { |
| 418 indexWriter.deleteDocuments(query); | |
| 1486 | 419 writeOp(OP_DELETE_DOCUMENTS); |
| 1465 | 420 log.writeQuery(query); |
| 421 } | |
| 422 | |
| 423 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException { | |
| 424 indexWriter.addDocument(storedFields); | |
| 1486 | 425 writeOp(OP_ADD_DOCUMENT); |
| 1465 | 426 log.writeMap(storedFields); |
| 427 } | |
| 428 | |
| 429 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { | |
| 430 indexWriter.updateDocument(keyFieldName,storedFields); | |
| 1486 | 431 writeOp(OP_UPDATE_DOCUMENT); |
| 1465 | 432 log.writeUTF(keyFieldName); |
| 433 log.writeMap(storedFields); | |
| 434 } | |
| 435 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
436 public synchronized void tag(String tag) throws IOException { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
437 writeOp(OP_TAG); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
438 log.writeUTF(tag); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
439 } |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
440 |
| 1465 | 441 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { |
| 1461 | 442 indexWriter.reindexDocuments(keyFieldName,query); |
| 443 } | |
| 444 | |
| 1486 | 445 private void writeOp(int op) throws IOException { |
| 1465 | 446 log.writeLong(System.currentTimeMillis()); |
| 447 log.writeByte(op); | |
| 448 } | |
| 449 | |
| 1549 | 450 public synchronized void playLogs(OpDoer opDoer) throws IOException { |
| 451 if( opDoer == null ) | |
| 1551 | 452 opDoer = new BasicOpDoer(indexWriter); |
| 1549 | 453 playLogs( logReaders(logs), opDoer ); |
| 1465 | 454 } |
| 455 | |
| 1548 | 456 private static LogInputStream[] logReaders(LogFile[] logs) throws IOException { |
| 457 LogInputStream[] logReaders = new LogInputStream[logs.length]; | |
| 458 for( int i=0; i<logs.length; i++ ) { | |
| 459 logReaders[i] = logs[i].input(); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
460 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
461 return logReaders; |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
462 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
463 |
| 1549 | 464 private static void playLogs(LogInputStream[] logReaders,OpDoer opDoer) |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
465 throws IOException |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
466 { |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
467 for( LogInputStream reader : logReaders ) { |
| 1549 | 468 playLog(reader,opDoer); |
| 1465 | 469 } |
| 1549 | 470 opDoer.commit(); |
| 1465 | 471 } |
| 472 | |
| 1549 | 473 private static void playLog(LogInputStream in,OpDoer opDoer) |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
474 throws IOException |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
475 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
476 while( in.available() > 0 ) { |
| 1549 | 477 playOp(in,opDoer); |
| 1465 | 478 } |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
479 in.close(); |
| 1465 | 480 } |
| 481 | |
| 1549 | 482 private static void playOp(LogInputStream in,OpDoer opDoer) |
| 1548 | 483 throws IOException |
| 484 { | |
| 1549 | 485 long time = in.readLong(); // time |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
486 int op = in.readByte(); |
| 1461 | 487 switch(op) { |
| 488 case OP_DELETE_ALL: | |
| 1549 | 489 opDoer.deleteAll(time); |
| 1548 | 490 return; |
| 1461 | 491 case OP_DELETE_DOCUMENTS: |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
492 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
493 Query query = in.readQuery(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
494 //System.out.println("OP_DELETE_DOCUMENTS "+query); |
| 1549 | 495 opDoer.deleteDocuments(time,query); |
| 1548 | 496 return; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
497 } |
| 1461 | 498 case OP_ADD_DOCUMENT: |
| 1465 | 499 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
500 Map storedFields = in.readMap(); |
| 1549 | 501 opDoer.addDocument(time,storedFields); |
| 1548 | 502 return; |
| 1465 | 503 } |
| 1461 | 504 case OP_UPDATE_DOCUMENT: |
| 1465 | 505 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
506 String keyFieldName = in.readUTF(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
507 Map storedFields = in.readMap(); |
| 1549 | 508 opDoer.updateDocument(time,keyFieldName,storedFields); |
| 1548 | 509 return; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
510 } |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
511 case OP_TAG: |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
512 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
513 String tag = in.readUTF(); |
| 1549 | 514 opDoer.tag(time,tag); |
| 1548 | 515 return; |
| 1465 | 516 } |
| 1461 | 517 default: |
| 518 throw new RuntimeException("invalid op "+op); | |
| 519 } | |
| 520 } | |
| 521 | |
| 1465 | 522 private static void dump(LuceneIndexWriter indexWriter) throws IOException { |
| 523 IndexReader reader = indexWriter.openReader(); | |
| 524 IndexSearcher searcher = new IndexSearcher(reader); | |
| 525 Query query = new MatchAllDocsQuery(); | |
| 526 TopDocs td = searcher.search(query,100); | |
| 527 System.out.println("totalHits = "+td.totalHits); | |
| 528 for( int i=0; i<td.scoreDocs.length; i++ ) { | |
| 529 Document doc = searcher.doc(td.scoreDocs[i].doc); | |
| 530 System.out.println(LuceneUtils.toMap(doc)); | |
| 1461 | 531 } |
| 1465 | 532 System.out.println(); |
| 533 reader.close(); | |
| 1461 | 534 } |
| 1465 | 535 |
| 1461 | 536 } |