Mercurial Hosting > luan
annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1548:736ec76bbf42
lucene log work
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Sun, 27 Sep 2020 22:07:18 -0600 |
| parents | 35601f15ecc3 |
| children | 41c32da4cbd1 |
| rev | line source |
|---|---|
| 1461 | 1 package goodjava.lucene.logging; |
| 2 | |
| 1465 | 3 import java.io.File; |
| 4 import java.io.RandomAccessFile; | |
| 5 import java.io.ByteArrayOutputStream; | |
| 6 import java.io.DataOutputStream; | |
| 7 import java.io.DataInputStream; | |
| 8 import java.io.FileInputStream; | |
| 1461 | 9 import java.io.IOException; |
| 10 import java.util.Map; | |
| 1465 | 11 import java.util.Set; |
| 12 import java.util.HashSet; | |
| 13 import java.util.Random; | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
14 import java.util.concurrent.TimeUnit; |
| 1465 | 15 import org.apache.lucene.document.Document; |
| 16 import org.apache.lucene.index.DirectoryReader; | |
| 17 import org.apache.lucene.index.IndexReader; | |
| 1539 | 18 import org.apache.lucene.index.IndexWriter; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
19 import org.apache.lucene.index.Term; |
| 1465 | 20 import org.apache.lucene.search.IndexSearcher; |
| 1461 | 21 import org.apache.lucene.search.Query; |
| 1465 | 22 import org.apache.lucene.search.MatchAllDocsQuery; |
| 23 import org.apache.lucene.search.TopDocs; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
24 import org.apache.lucene.search.PrefixQuery; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
25 import org.apache.lucene.search.SortField; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
26 import org.apache.lucene.search.Sort; |
| 1465 | 27 import org.apache.lucene.store.Directory; |
| 28 import org.apache.lucene.store.FSDirectory; | |
| 1473 | 29 import goodjava.io.IoUtils; |
| 1548 | 30 import goodjava.lucene.api.GoodWriter; |
| 1461 | 31 import goodjava.lucene.api.GoodIndexWriter; |
| 1465 | 32 import goodjava.lucene.api.LuceneIndexWriter; |
| 33 import goodjava.lucene.api.GoodCollector; | |
| 34 import goodjava.lucene.api.LuceneUtils; | |
| 35 import goodjava.logging.Logger; | |
| 36 import goodjava.logging.LoggerFactory; | |
| 1461 | 37 |
| 38 | |
| 1488 | 39 public class LoggingIndexWriter implements GoodIndexWriter { |
| 1465 | 40 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class); |
| 1548 | 41 private static final int version = 2; |
| 1461 | 42 private static final int OP_DELETE_ALL = 1; |
| 43 private static final int OP_DELETE_DOCUMENTS = 2; | |
| 44 private static final int OP_ADD_DOCUMENT = 3; | |
| 45 private static final int OP_UPDATE_DOCUMENT = 4; | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
46 private static final int OP_TAG = 5; |
| 1465 | 47 private static final Random rnd = new Random(); |
| 1461 | 48 |
| 1465 | 49 public final LuceneIndexWriter indexWriter; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
50 public boolean wasCreated; |
| 1465 | 51 private final File logDir; |
| 1548 | 52 private final long logTime; |
| 53 protected final LogFile[] logs = new LogFile[3]; | |
| 1486 | 54 private LogOutputStream log; |
| 1465 | 55 private final File index; |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
56 private final SemaphoreLock mergeLock = new SemaphoreLock(); |
| 1461 | 57 |
| 1548 | 58 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime) |
| 59 throws IOException | |
| 60 { | |
| 1461 | 61 this.indexWriter = indexWriter; |
| 1465 | 62 this.logDir = logDir; |
| 1548 | 63 this.logTime = logTime; |
| 1501 | 64 IoUtils.mkdirs(logDir); |
| 1465 | 65 if( !logDir.isDirectory() ) |
| 66 throw new RuntimeException(); | |
| 67 index = new File(logDir,"index"); | |
| 68 if( index.exists() ) { | |
| 69 DataInputStream dis = new DataInputStream(new FileInputStream(index)); | |
| 70 try { | |
| 71 if( dis.readInt() == version ) { | |
| 1548 | 72 for( int i=0; i<logs.length; i++ ) { |
| 1465 | 73 File file = new File( logDir, dis.readUTF() ); |
| 1548 | 74 logs[i] = new LogFile(file); |
| 1465 | 75 } |
| 76 deleteUnusedFiles(); | |
| 1486 | 77 setLog(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
78 wasCreated = false; |
| 1465 | 79 return; |
| 80 } | |
| 81 } finally { | |
| 82 dis.close(); | |
| 83 } | |
| 84 } | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
85 newLogs(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
86 wasCreated = true; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
87 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
88 |
| 1548 | 89 public IndexReader openReader() throws IOException { |
| 90 return indexWriter.openReader(); | |
| 91 } | |
| 92 | |
| 1539 | 93 public IndexWriter getLuceneIndexWriter() { |
| 94 return indexWriter.getLuceneIndexWriter(); | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
95 } |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
96 |
| 1486 | 97 private void setLog() throws IOException { |
| 98 if( log != null ) | |
| 99 log.close(); | |
| 1548 | 100 log = logs[2].output(); |
| 1486 | 101 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
102 /* |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
103 public synchronized boolean isMerging() { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
104 return mergeLock.isLocked(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
105 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
106 */ |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
107 private void getMergeLock() { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
108 try { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
109 if( !mergeLock.tryLock(1,TimeUnit.MINUTES) ) |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
110 throw new RuntimeException("failed to acquire lock"); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
111 } catch(InterruptedException e) { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
112 throw new RuntimeException(e); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
113 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
114 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
115 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
116 public synchronized void newLogs() throws IOException { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
117 getMergeLock(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
118 try { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
119 newLogs2(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
120 } finally { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
121 mergeLock.unlock(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
122 } |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
123 } |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
124 |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
125 private void newLogs2() throws IOException { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
126 logger.info("building new logs"); |
| 1548 | 127 for( int i=0; i<logs.length; i++ ) { |
| 128 logs[i] = newLogFile(); | |
| 1465 | 129 } |
| 1548 | 130 LogOutputStream log = logs[0].output(); |
| 131 logLucene( System.currentTimeMillis(), log, indexWriter ); | |
| 132 log.close(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
133 writeIndex(); |
| 1486 | 134 setLog(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
135 logger.info("done building new logs"); |
| 1461 | 136 } |
| 137 | |
| 1548 | 138 public synchronized void logLucene() |
| 139 throws IOException | |
| 140 { | |
| 141 //log.rollback(); ? | |
| 142 logLucene( System.currentTimeMillis(), log, indexWriter ); | |
| 143 } | |
| 144 | |
| 145 private static void logLucene(long time,LogOutputStream log,LuceneIndexWriter indexWriter) | |
| 146 throws IOException | |
| 147 { | |
| 148 log.writeLong(time); | |
| 149 log.writeByte(OP_DELETE_ALL); | |
| 1465 | 150 IndexReader reader = indexWriter.openReader(); |
| 151 final IndexSearcher searcher = new IndexSearcher(reader); | |
| 152 Query query = new MatchAllDocsQuery(); | |
| 153 searcher.search( query, new GoodCollector(){ | |
| 154 public void collectDoc(int iDoc) throws IOException { | |
| 155 Document doc = searcher.doc(iDoc); | |
| 156 Map<String,Object> storedFields = LuceneUtils.toMap(doc); | |
| 157 log.writeLong(time); | |
| 158 log.writeByte(OP_ADD_DOCUMENT); | |
| 159 log.writeMap(storedFields); | |
| 160 } | |
| 161 }); | |
| 162 reader.close(); | |
| 163 log.commit(); | |
| 164 } | |
| 165 | |
| 166 private LogFile newLogFile() throws IOException { | |
| 167 File file; | |
| 168 do { | |
| 169 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); | |
| 170 } while( file.exists() ); | |
| 1486 | 171 return new LogFile(file); |
| 1461 | 172 } |
| 173 | |
| 1473 | 174 private void deleteUnusedFiles() throws IOException { |
| 1499 | 175 deleteUnusedFiles(logs,index); |
| 176 } | |
| 177 | |
| 1548 | 178 private static void deleteUnusedFiles(LogFile[] logs,File index) throws IOException { |
| 1465 | 179 Set<String> used = new HashSet<String>(); |
| 180 used.add( index.getName() ); | |
| 181 for( LogFile lf : logs ) { | |
| 182 used.add( lf.file.getName() ); | |
| 183 } | |
| 1499 | 184 for( File f : index.getParentFile().listFiles() ) { |
| 1465 | 185 if( !used.contains(f.getName()) ) { |
| 1475 | 186 IoUtils.deleteRecursively(f); |
| 1465 | 187 } |
| 188 } | |
| 1461 | 189 } |
| 190 | |
| 1465 | 191 private void writeIndex() throws IOException { |
| 1499 | 192 writeIndex(logs,index); |
| 193 } | |
| 194 | |
| 1548 | 195 public static void writeIndex(LogFile[] logs,File index) throws IOException { |
| 196 if( logs.length != 3 ) | |
| 197 throw new RuntimeException(); | |
| 1465 | 198 ByteArrayOutputStream baos = new ByteArrayOutputStream(); |
| 199 DataOutputStream dos = new DataOutputStream(baos); | |
| 200 dos.writeInt(version); | |
| 201 for( LogFile lf : logs ) { | |
| 202 String fileName = lf.file.getName(); | |
| 203 dos.writeUTF(fileName); | |
| 204 } | |
| 205 dos.close(); | |
| 206 RandomAccessFile raf = new RandomAccessFile( index, "rwd" ); | |
| 207 raf.write( baos.toByteArray() ); | |
| 208 raf.close(); | |
| 1499 | 209 deleteUnusedFiles(logs,index); |
| 210 //logger.info("writeIndex "+logs.toString()); | |
| 1461 | 211 } |
| 212 | |
| 1465 | 213 private void mergeLogs() throws IOException { |
| 1548 | 214 logger.info("merge"); |
| 215 if( !mergeLock.isLocked() ) { | |
| 216 logger.error("merge without lock"); | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
217 return; |
| 1548 | 218 } |
| 219 LogFile first = logs[0]; | |
| 220 LogFile second = logs[1]; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
221 long lastTime = second.file.lastModified(); |
| 1465 | 222 File dirFile = new File(logDir,"merge"); |
| 223 if( dirFile.exists() ) | |
| 224 throw new RuntimeException(); | |
| 225 Directory dir = FSDirectory.open(dirFile); | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
226 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
| 1548 | 227 playLog( first.input(), mergeWriter ); |
| 228 playLog( second.input(), mergeWriter ); | |
| 1465 | 229 mergeWriter.commit(); |
| 230 LogFile merge = newLogFile(); | |
| 1548 | 231 LogOutputStream log = merge.output(); |
| 232 logLucene( lastTime, log, mergeWriter ); | |
| 233 log.close(); | |
| 1465 | 234 mergeWriter.close(); |
| 235 synchronized(this) { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
236 //check(); |
| 1548 | 237 logs[0] = merge; |
| 238 logs[1] = logs[2]; | |
| 239 logs[2] = newLogFile(); | |
| 1465 | 240 writeIndex(); |
| 1548 | 241 setLog(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
242 //check(null); |
| 1465 | 243 } |
| 1461 | 244 } |
| 1465 | 245 private final Runnable mergeLogs = new Runnable() { public void run() { |
| 246 try { | |
| 247 mergeLogs(); | |
| 248 } catch(IOException e) { | |
| 249 throw new RuntimeException(e); | |
| 250 } finally { | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
251 mergeLock.unlock(); |
| 1465 | 252 } |
| 253 } }; | |
| 1461 | 254 |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
255 private static class DocIter { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
256 final IndexReader reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
257 final TopDocs td; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
258 final int n; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
259 int i = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
260 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
261 DocIter(IndexReader reader,Query query,Sort sort) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
262 this.reader = reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
263 IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
264 this.td = searcher.search(query,10000000,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
265 this.n = td.scoreDocs.length; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
266 if( td.totalHits != n ) |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
267 throw new RuntimeException(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
268 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
269 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
270 Document next() throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
271 return i < n ? reader.document(td.scoreDocs[i++].doc) : null; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
272 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
273 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
274 |
| 1487 | 275 private volatile boolean isChecking = false; |
| 276 | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
277 public boolean check(SortField sortField) throws IOException { |
| 1487 | 278 if( isChecking ) |
| 279 throw new RuntimeException("another check is running"); | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
280 isChecking = true; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
281 try { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
282 return doCheck(sortField); |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
283 } finally { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
284 isChecking = false; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
285 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
286 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
287 |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
288 protected boolean doCheck(SortField sortField) throws IOException { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
289 boolean ok = true; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
290 IndexReader indexReader; |
| 1548 | 291 LogInputStream[] logReaders; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
292 synchronized(this) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
293 indexReader = indexWriter.openReader(); |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
294 logReaders = logReaders(logs); |
| 1465 | 295 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
296 try { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
297 //logger.info("check start"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
298 indexWriter.check(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
299 File dirFile = new File(logDir,"check"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
300 IoUtils.deleteRecursively(dirFile); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
301 Directory dir = FSDirectory.open(dirFile); |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
302 LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
| 1548 | 303 playLogs(logReaders,checkWriter); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
304 //logger.info("check lucene"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
305 IndexReader checkReader = checkWriter.openReader(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
306 int nCheck = checkReader.numDocs(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
307 int nOrig = indexReader.numDocs(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
308 if( nCheck != nOrig ) { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
309 logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
310 ok = false; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
311 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
312 if( sortField == null ) { |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
313 if( ok && hash(indexReader) != hash(checkReader) ) { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
314 logger.error("hash mismatch"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
315 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
316 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
317 } else { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
318 Sort sort = new Sort(sortField); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
319 String sortFieldName = sortField.getField(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
320 Query query = new PrefixQuery(new Term(sortFieldName)); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
321 DocIter origIter = new DocIter(indexReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
322 DocIter checkIter = new DocIter(checkReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
323 Map<String,Object> origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
324 Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
325 while( origFields!=null && checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
326 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
327 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
328 int cmp = origFld.compareTo(checkFld); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
329 if( cmp==0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
330 if( !origFields.equals(checkFields) ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
331 logger.error(sortFieldName+" "+origFld+" not equal"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
332 logger.error("lucene = "+origFields); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
333 logger.error("logs = "+checkFields); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
334 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
335 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
336 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
337 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
338 } else if( cmp < 0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
339 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
340 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
341 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
342 } else { // > |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
343 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
344 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
345 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
346 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
347 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
348 while( origFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
349 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
350 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
351 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
352 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
353 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
354 while( checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
355 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
356 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
357 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
358 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
359 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
360 //logger.info("check done"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
361 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
362 checkReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
363 checkWriter.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
364 IoUtils.deleteRecursively(dirFile); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
365 //logger.info("check done"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
366 } finally { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
367 indexReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
368 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
369 return ok; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
370 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
371 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
372 private static abstract class HashCollector extends GoodCollector { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
373 int total = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
374 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
375 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
376 private static int hash(IndexReader reader) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
377 final IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
378 Query query = new MatchAllDocsQuery(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
379 HashCollector col = new HashCollector() { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
380 public void collectDoc(int iDoc) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
381 Document doc = searcher.doc(iDoc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
382 Map<String,Object> storedFields = LuceneUtils.toMap(doc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
383 total += storedFields.hashCode(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
384 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
385 }; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
386 searcher.search(query,col); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
387 return col.total; |
| 1461 | 388 } |
| 389 | |
| 1465 | 390 public synchronized void close() throws IOException { |
| 391 indexWriter.close(); | |
| 392 log.commit(); | |
| 1486 | 393 log.close(); |
| 1465 | 394 } |
| 395 | |
| 396 public synchronized void commit() throws IOException { | |
| 397 indexWriter.commit(); | |
| 398 log.commit(); | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
399 if( mergeLock.isLocked() ) |
| 1465 | 400 return; |
| 1548 | 401 if( logs[1].file.lastModified() < System.currentTimeMillis() - logTime ) { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
402 getMergeLock(); |
| 1504 | 403 new Thread(mergeLogs).start(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
404 // mergeLogs.run(); |
| 1465 | 405 } |
| 1461 | 406 } |
| 407 | |
| 1465 | 408 public synchronized void rollback() throws IOException { |
| 409 indexWriter.rollback(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
410 log.rollback(); |
| 1465 | 411 } |
| 412 | |
| 413 public synchronized void deleteAll() throws IOException { | |
| 414 indexWriter.deleteAll(); | |
| 1486 | 415 writeOp(OP_DELETE_ALL); |
| 1461 | 416 } |
| 417 | |
| 1465 | 418 public synchronized void deleteDocuments(Query query) throws IOException { |
| 419 indexWriter.deleteDocuments(query); | |
| 1486 | 420 writeOp(OP_DELETE_DOCUMENTS); |
| 1465 | 421 log.writeQuery(query); |
| 422 } | |
| 423 | |
| 424 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException { | |
| 425 indexWriter.addDocument(storedFields); | |
| 1486 | 426 writeOp(OP_ADD_DOCUMENT); |
| 1465 | 427 log.writeMap(storedFields); |
| 428 } | |
| 429 | |
| 430 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { | |
| 431 indexWriter.updateDocument(keyFieldName,storedFields); | |
| 1486 | 432 writeOp(OP_UPDATE_DOCUMENT); |
| 1465 | 433 log.writeUTF(keyFieldName); |
| 434 log.writeMap(storedFields); | |
| 435 } | |
| 436 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
437 public synchronized void tag(String tag) throws IOException { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
438 writeOp(OP_TAG); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
439 log.writeUTF(tag); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
440 } |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
441 |
| 1465 | 442 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { |
| 1461 | 443 indexWriter.reindexDocuments(keyFieldName,query); |
| 444 } | |
| 445 | |
| 1486 | 446 private void writeOp(int op) throws IOException { |
| 1465 | 447 log.writeLong(System.currentTimeMillis()); |
| 448 log.writeByte(op); | |
| 449 } | |
| 450 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
451 // return whether stopped at tag |
| 1548 | 452 public synchronized void playLogs(GoodWriter writer) throws IOException { |
| 453 if( writer == null ) | |
| 454 writer = indexWriter; | |
| 455 playLogs( logReaders(logs), writer ); | |
| 1465 | 456 } |
| 457 | |
| 1548 | 458 private static LogInputStream[] logReaders(LogFile[] logs) throws IOException { |
| 459 LogInputStream[] logReaders = new LogInputStream[logs.length]; | |
| 460 for( int i=0; i<logs.length; i++ ) { | |
| 461 logReaders[i] = logs[i].input(); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
462 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
463 return logReaders; |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
464 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
465 |
| 1548 | 466 private static void playLogs(LogInputStream[] logReaders,GoodWriter indexWriter) |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
467 throws IOException |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
468 { |
| 1465 | 469 if( numDocs(indexWriter) != 0 ) |
| 470 throw new RuntimeException ("not empty"); | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
471 for( LogInputStream reader : logReaders ) { |
| 1548 | 472 playLog(reader,indexWriter); |
| 1465 | 473 } |
| 474 indexWriter.commit(); | |
| 475 } | |
| 476 | |
| 1548 | 477 private static int numDocs(GoodWriter indexWriter) throws IOException { |
| 1465 | 478 IndexReader reader = indexWriter.openReader(); |
| 479 int n = reader.numDocs(); | |
| 480 reader.close(); | |
| 481 return n; | |
| 482 } | |
| 483 | |
| 1548 | 484 private static void playLog(LogInputStream in,GoodWriter indexWriter) |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
485 throws IOException |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
486 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
487 while( in.available() > 0 ) { |
| 1548 | 488 playOp(in,indexWriter); |
| 1465 | 489 } |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
490 in.close(); |
| 1465 | 491 } |
| 492 | |
| 1548 | 493 private static void playOp(LogInputStream in,GoodWriter indexWriter) |
| 494 throws IOException | |
| 495 { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
496 in.readLong(); // time |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
497 int op = in.readByte(); |
| 1461 | 498 switch(op) { |
| 499 case OP_DELETE_ALL: | |
| 500 indexWriter.deleteAll(); | |
| 1548 | 501 return; |
| 1461 | 502 case OP_DELETE_DOCUMENTS: |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
503 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
504 Query query = in.readQuery(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
505 //System.out.println("OP_DELETE_DOCUMENTS "+query); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
506 indexWriter.deleteDocuments(query); |
| 1548 | 507 return; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
508 } |
| 1461 | 509 case OP_ADD_DOCUMENT: |
| 1465 | 510 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
511 Map storedFields = in.readMap(); |
| 1465 | 512 indexWriter.addDocument(storedFields); |
| 1548 | 513 return; |
| 1465 | 514 } |
| 1461 | 515 case OP_UPDATE_DOCUMENT: |
| 1465 | 516 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
517 String keyFieldName = in.readUTF(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
518 Map storedFields = in.readMap(); |
| 1465 | 519 indexWriter.updateDocument(keyFieldName,storedFields); |
| 1548 | 520 return; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
521 } |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
522 case OP_TAG: |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
523 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
524 String tag = in.readUTF(); |
| 1548 | 525 indexWriter.tag(tag); |
| 526 return; | |
| 1465 | 527 } |
| 1461 | 528 default: |
| 529 throw new RuntimeException("invalid op "+op); | |
| 530 } | |
| 531 } | |
| 532 | |
| 1465 | 533 private static void dump(LuceneIndexWriter indexWriter) throws IOException { |
| 534 IndexReader reader = indexWriter.openReader(); | |
| 535 IndexSearcher searcher = new IndexSearcher(reader); | |
| 536 Query query = new MatchAllDocsQuery(); | |
| 537 TopDocs td = searcher.search(query,100); | |
| 538 System.out.println("totalHits = "+td.totalHits); | |
| 539 for( int i=0; i<td.scoreDocs.length; i++ ) { | |
| 540 Document doc = searcher.doc(td.scoreDocs[i].doc); | |
| 541 System.out.println(LuceneUtils.toMap(doc)); | |
| 1461 | 542 } |
| 1465 | 543 System.out.println(); |
| 544 reader.close(); | |
| 1461 | 545 } |
| 1465 | 546 |
| 1461 | 547 } |