Mercurial Hosting > luan
annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1544:35601f15ecc3
add lucene log tag and restore_from_log
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Sun, 20 Sep 2020 20:36:55 -0600 |
| parents | c27dc6af87ca |
| children | 736ec76bbf42 |
| rev | line source |
|---|---|
| 1461 | 1 package goodjava.lucene.logging; |
| 2 | |
| 1465 | 3 import java.io.File; |
| 4 import java.io.RandomAccessFile; | |
| 5 import java.io.ByteArrayOutputStream; | |
| 6 import java.io.DataOutputStream; | |
| 7 import java.io.DataInputStream; | |
| 8 import java.io.FileInputStream; | |
| 1461 | 9 import java.io.IOException; |
| 10 import java.util.Map; | |
| 1465 | 11 import java.util.Set; |
| 12 import java.util.HashSet; | |
| 13 import java.util.List; | |
| 14 import java.util.ArrayList; | |
| 15 import java.util.Random; | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
16 import java.util.concurrent.TimeUnit; |
| 1465 | 17 import org.apache.lucene.document.Document; |
| 18 import org.apache.lucene.index.DirectoryReader; | |
| 19 import org.apache.lucene.index.IndexReader; | |
| 1539 | 20 import org.apache.lucene.index.IndexWriter; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
21 import org.apache.lucene.index.Term; |
| 1465 | 22 import org.apache.lucene.search.IndexSearcher; |
| 1461 | 23 import org.apache.lucene.search.Query; |
| 1465 | 24 import org.apache.lucene.search.MatchAllDocsQuery; |
| 25 import org.apache.lucene.search.TopDocs; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
26 import org.apache.lucene.search.PrefixQuery; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
27 import org.apache.lucene.search.SortField; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
28 import org.apache.lucene.search.Sort; |
| 1465 | 29 import org.apache.lucene.store.Directory; |
| 30 import org.apache.lucene.store.FSDirectory; | |
| 1473 | 31 import goodjava.io.IoUtils; |
| 1461 | 32 import goodjava.lucene.api.GoodIndexWriter; |
| 1465 | 33 import goodjava.lucene.api.LuceneIndexWriter; |
| 34 import goodjava.lucene.api.GoodCollector; | |
| 35 import goodjava.lucene.api.LuceneUtils; | |
| 36 import goodjava.logging.Logger; | |
| 37 import goodjava.logging.LoggerFactory; | |
| 1461 | 38 |
| 39 | |
| 1488 | 40 public class LoggingIndexWriter implements GoodIndexWriter { |
| 1465 | 41 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class); |
| 42 private static final int version = 1; | |
| 1461 | 43 private static final int OP_DELETE_ALL = 1; |
| 44 private static final int OP_DELETE_DOCUMENTS = 2; | |
| 45 private static final int OP_ADD_DOCUMENT = 3; | |
| 46 private static final int OP_UPDATE_DOCUMENT = 4; | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
47 private static final int OP_TAG = 5; |
| 1465 | 48 private static final Random rnd = new Random(); |
| 1461 | 49 |
| 1465 | 50 public final LuceneIndexWriter indexWriter; |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
51 public boolean wasCreated; |
| 1465 | 52 private final File logDir; |
| 1488 | 53 protected final List<LogFile> logs = new ArrayList<LogFile>(); |
| 1486 | 54 private LogOutputStream log; |
| 1465 | 55 private final File index; |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
56 private final SemaphoreLock mergeLock = new SemaphoreLock(); |
| 1461 | 57 |
| 1465 | 58 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir) throws IOException { |
| 1461 | 59 this.indexWriter = indexWriter; |
| 1465 | 60 this.logDir = logDir; |
| 1501 | 61 IoUtils.mkdirs(logDir); |
| 1465 | 62 if( !logDir.isDirectory() ) |
| 63 throw new RuntimeException(); | |
| 64 index = new File(logDir,"index"); | |
| 65 if( index.exists() ) { | |
| 66 DataInputStream dis = new DataInputStream(new FileInputStream(index)); | |
| 67 try { | |
| 68 if( dis.readInt() == version ) { | |
| 69 final int n = dis.readInt(); | |
| 70 for( int i=0; i<n; i++ ) { | |
| 71 File file = new File( logDir, dis.readUTF() ); | |
| 1486 | 72 logs.add( new LogFile(file) ); |
| 1465 | 73 } |
| 74 deleteUnusedFiles(); | |
| 1486 | 75 setLog(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
76 wasCreated = false; |
| 1465 | 77 return; |
| 78 } | |
| 79 } finally { | |
| 80 dis.close(); | |
| 81 } | |
| 82 } | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
83 newLogs(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
84 wasCreated = true; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
85 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
86 |
| 1539 | 87 public IndexWriter getLuceneIndexWriter() { |
| 88 return indexWriter.getLuceneIndexWriter(); | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
89 } |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
90 |
| 1486 | 91 private void setLog() throws IOException { |
| 92 if( log != null ) | |
| 93 log.close(); | |
| 94 log = logs.get(logs.size()-1).output(); | |
| 95 } | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
96 /* |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
97 public synchronized boolean isMerging() { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
98 return mergeLock.isLocked(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
99 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
100 */ |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
101 private void getMergeLock() { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
102 try { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
103 if( !mergeLock.tryLock(1,TimeUnit.MINUTES) ) |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
104 throw new RuntimeException("failed to acquire lock"); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
105 } catch(InterruptedException e) { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
106 throw new RuntimeException(e); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
107 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
108 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
109 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
110 public synchronized void newLogs() throws IOException { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
111 getMergeLock(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
112 try { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
113 newLogs2(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
114 } finally { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
115 mergeLock.unlock(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
116 } |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
117 } |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
118 |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
119 private void newLogs2() throws IOException { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
120 logger.info("building new logs"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
121 logs.clear(); |
| 1465 | 122 for( int i=0; i<2; i++ ) { |
| 123 logs.add( newLogFile() ); | |
| 124 } | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
125 logLucene( System.currentTimeMillis(), logs.get(0), indexWriter ); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
126 writeIndex(); |
| 1486 | 127 setLog(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
128 logger.info("done building new logs"); |
| 1461 | 129 } |
| 130 | |
| 1486 | 131 private static void logLucene(long time,LogFile logLucene,LuceneIndexWriter indexWriter) throws IOException { |
| 132 LogOutputStream log = logLucene.output(); | |
| 1465 | 133 IndexReader reader = indexWriter.openReader(); |
| 134 final IndexSearcher searcher = new IndexSearcher(reader); | |
| 135 Query query = new MatchAllDocsQuery(); | |
| 136 searcher.search( query, new GoodCollector(){ | |
| 137 public void collectDoc(int iDoc) throws IOException { | |
| 138 Document doc = searcher.doc(iDoc); | |
| 139 Map<String,Object> storedFields = LuceneUtils.toMap(doc); | |
| 140 log.writeLong(time); | |
| 141 log.writeByte(OP_ADD_DOCUMENT); | |
| 142 log.writeMap(storedFields); | |
| 143 } | |
| 144 }); | |
| 145 reader.close(); | |
| 146 log.commit(); | |
| 1486 | 147 log.close(); |
| 1465 | 148 } |
| 149 | |
| 150 private LogFile newLogFile() throws IOException { | |
| 151 File file; | |
| 152 do { | |
| 153 file = new File(logDir,"_"+rnd.nextInt(100)+".log"); | |
| 154 } while( file.exists() ); | |
| 1486 | 155 return new LogFile(file); |
| 1461 | 156 } |
| 157 | |
| 1473 | 158 private void deleteUnusedFiles() throws IOException { |
| 1499 | 159 deleteUnusedFiles(logs,index); |
| 160 } | |
| 161 | |
| 162 private static void deleteUnusedFiles(List<LogFile> logs,File index) throws IOException { | |
| 1465 | 163 Set<String> used = new HashSet<String>(); |
| 164 used.add( index.getName() ); | |
| 165 for( LogFile lf : logs ) { | |
| 166 used.add( lf.file.getName() ); | |
| 167 } | |
| 1499 | 168 for( File f : index.getParentFile().listFiles() ) { |
| 1465 | 169 if( !used.contains(f.getName()) ) { |
| 1475 | 170 IoUtils.deleteRecursively(f); |
| 1465 | 171 } |
| 172 } | |
| 1461 | 173 } |
| 174 | |
| 1465 | 175 private void writeIndex() throws IOException { |
| 1499 | 176 writeIndex(logs,index); |
| 177 } | |
| 178 | |
| 179 public static void writeIndex(List<LogFile> logs,File index) throws IOException { | |
| 1465 | 180 ByteArrayOutputStream baos = new ByteArrayOutputStream(); |
| 181 DataOutputStream dos = new DataOutputStream(baos); | |
| 182 dos.writeInt(version); | |
| 183 dos.writeInt(logs.size()); | |
| 184 for( LogFile lf : logs ) { | |
| 185 String fileName = lf.file.getName(); | |
| 186 dos.writeUTF(fileName); | |
| 187 } | |
| 188 dos.close(); | |
| 189 RandomAccessFile raf = new RandomAccessFile( index, "rwd" ); | |
| 190 raf.write( baos.toByteArray() ); | |
| 191 raf.close(); | |
| 1499 | 192 deleteUnusedFiles(logs,index); |
| 193 //logger.info("writeIndex "+logs.toString()); | |
| 1461 | 194 } |
| 195 | |
| 1465 | 196 private void mergeLogs() throws IOException { |
| 1512 | 197 //logger.info("merge"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
198 if( logs.size() <= 3 ) |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
199 return; |
| 1465 | 200 LogFile first = logs.get(0); |
| 201 LogFile second = logs.get(1); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
202 long lastTime = second.file.lastModified(); |
| 1465 | 203 File dirFile = new File(logDir,"merge"); |
| 204 if( dirFile.exists() ) | |
| 205 throw new RuntimeException(); | |
| 206 Directory dir = FSDirectory.open(dirFile); | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
207 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
208 playLog( first.input(), mergeWriter, null ); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
209 playLog( second.input(), mergeWriter, null ); |
| 1465 | 210 mergeWriter.commit(); |
| 211 LogFile merge = newLogFile(); | |
| 212 logLucene( lastTime, merge, mergeWriter ); | |
| 213 mergeWriter.close(); | |
| 214 synchronized(this) { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
215 //check(); |
| 1465 | 216 logs.remove(0); |
| 217 logs.set(0,merge); | |
| 218 writeIndex(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
219 //check(null); |
| 1465 | 220 } |
| 1461 | 221 } |
| 1465 | 222 private final Runnable mergeLogs = new Runnable() { public void run() { |
| 223 try { | |
| 224 mergeLogs(); | |
| 225 } catch(IOException e) { | |
| 226 throw new RuntimeException(e); | |
| 227 } finally { | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
228 mergeLock.unlock(); |
| 1465 | 229 } |
| 230 } }; | |
| 1461 | 231 |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
232 private static class DocIter { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
233 final IndexReader reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
234 final TopDocs td; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
235 final int n; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
236 int i = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
237 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
238 DocIter(IndexReader reader,Query query,Sort sort) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
239 this.reader = reader; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
240 IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
241 this.td = searcher.search(query,10000000,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
242 this.n = td.scoreDocs.length; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
243 if( td.totalHits != n ) |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
244 throw new RuntimeException(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
245 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
246 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
247 Document next() throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
248 return i < n ? reader.document(td.scoreDocs[i++].doc) : null; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
249 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
250 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
251 |
| 1487 | 252 private volatile boolean isChecking = false; |
| 253 | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
254 public boolean check(SortField sortField) throws IOException { |
| 1487 | 255 if( isChecking ) |
| 256 throw new RuntimeException("another check is running"); | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
257 isChecking = true; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
258 try { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
259 return doCheck(sortField); |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
260 } finally { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
261 isChecking = false; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
262 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
263 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1504
diff
changeset
|
264 |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
265 protected boolean doCheck(SortField sortField) throws IOException { |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
266 boolean ok = true; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
267 IndexReader indexReader; |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
268 List<LogInputStream> logReaders; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
269 synchronized(this) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
270 indexReader = indexWriter.openReader(); |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
271 logReaders = logReaders(logs); |
| 1465 | 272 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
273 try { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
274 //logger.info("check start"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
275 indexWriter.check(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
276 File dirFile = new File(logDir,"check"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
277 IoUtils.deleteRecursively(dirFile); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
278 Directory dir = FSDirectory.open(dirFile); |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
279 LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig ); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
280 playLogs(logReaders,checkWriter,null); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
281 //logger.info("check lucene"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
282 IndexReader checkReader = checkWriter.openReader(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
283 int nCheck = checkReader.numDocs(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
284 int nOrig = indexReader.numDocs(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
285 if( nCheck != nOrig ) { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
286 logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
287 ok = false; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
288 } |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
289 if( sortField == null ) { |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
290 if( ok && hash(indexReader) != hash(checkReader) ) { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
291 logger.error("hash mismatch"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
292 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
293 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
294 } else { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
295 Sort sort = new Sort(sortField); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
296 String sortFieldName = sortField.getField(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
297 Query query = new PrefixQuery(new Term(sortFieldName)); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
298 DocIter origIter = new DocIter(indexReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
299 DocIter checkIter = new DocIter(checkReader,query,sort); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
300 Map<String,Object> origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
301 Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
302 while( origFields!=null && checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
303 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
304 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
305 int cmp = origFld.compareTo(checkFld); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
306 if( cmp==0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
307 if( !origFields.equals(checkFields) ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
308 logger.error(sortFieldName+" "+origFld+" not equal"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
309 logger.error("lucene = "+origFields); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
310 logger.error("logs = "+checkFields); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
311 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
312 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
313 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
314 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
315 } else if( cmp < 0 ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
316 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
317 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
318 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
319 } else { // > |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
320 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
321 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
322 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
323 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
324 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
325 while( origFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
326 Comparable origFld = (Comparable)origFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
327 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
328 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
329 origFields = LuceneUtils.toMap(origIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
330 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
331 while( checkFields!=null ) { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
332 Comparable checkFld = (Comparable)checkFields.get(sortFieldName); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
333 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene"); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
334 ok = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
335 checkFields = LuceneUtils.toMap(checkIter.next()); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
336 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
337 //logger.info("check done"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
338 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
339 checkReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
340 checkWriter.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
341 IoUtils.deleteRecursively(dirFile); |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
342 //logger.info("check done"); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
343 } finally { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
344 indexReader.close(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
345 } |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
346 return ok; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
347 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
348 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
349 private static abstract class HashCollector extends GoodCollector { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
350 int total = 0; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
351 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
352 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
353 private static int hash(IndexReader reader) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
354 final IndexSearcher searcher = new IndexSearcher(reader); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
355 Query query = new MatchAllDocsQuery(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
356 HashCollector col = new HashCollector() { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
357 public void collectDoc(int iDoc) throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
358 Document doc = searcher.doc(iDoc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
359 Map<String,Object> storedFields = LuceneUtils.toMap(doc); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
360 total += storedFields.hashCode(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
361 } |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
362 }; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
363 searcher.search(query,col); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
364 return col.total; |
| 1461 | 365 } |
| 366 | |
| 1465 | 367 public synchronized void close() throws IOException { |
| 368 indexWriter.close(); | |
| 369 log.commit(); | |
| 1486 | 370 log.close(); |
| 1465 | 371 } |
| 372 | |
| 373 public synchronized void commit() throws IOException { | |
| 374 indexWriter.commit(); | |
| 375 log.commit(); | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
376 if( mergeLock.isLocked() ) |
| 1465 | 377 return; |
| 1486 | 378 if( log.logFile.end() > logs.get(0).end() ) { |
| 1465 | 379 logs.add( newLogFile() ); |
| 380 writeIndex(); | |
| 1486 | 381 setLog(); |
| 1465 | 382 } |
| 383 if( logs.size() > 3 ) { | |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1528
diff
changeset
|
384 getMergeLock(); |
| 1504 | 385 new Thread(mergeLogs).start(); |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
386 // mergeLogs.run(); |
| 1465 | 387 } |
| 1461 | 388 } |
| 389 | |
| 1465 | 390 public synchronized void rollback() throws IOException { |
| 391 indexWriter.rollback(); | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
392 log.rollback(); |
| 1465 | 393 } |
| 394 | |
| 395 public synchronized void deleteAll() throws IOException { | |
| 396 indexWriter.deleteAll(); | |
| 1486 | 397 writeOp(OP_DELETE_ALL); |
| 1461 | 398 } |
| 399 | |
| 1465 | 400 public synchronized void deleteDocuments(Query query) throws IOException { |
| 401 indexWriter.deleteDocuments(query); | |
| 1486 | 402 writeOp(OP_DELETE_DOCUMENTS); |
| 1465 | 403 log.writeQuery(query); |
| 404 } | |
| 405 | |
| 406 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException { | |
| 407 indexWriter.addDocument(storedFields); | |
| 1486 | 408 writeOp(OP_ADD_DOCUMENT); |
| 1465 | 409 log.writeMap(storedFields); |
| 410 } | |
| 411 | |
| 412 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { | |
| 413 indexWriter.updateDocument(keyFieldName,storedFields); | |
| 1486 | 414 writeOp(OP_UPDATE_DOCUMENT); |
| 1465 | 415 log.writeUTF(keyFieldName); |
| 416 log.writeMap(storedFields); | |
| 417 } | |
| 418 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
419 public synchronized void tag(String tag) throws IOException { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
420 writeOp(OP_TAG); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
421 log.writeUTF(tag); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
422 } |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
423 |
| 1465 | 424 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException { |
| 1461 | 425 indexWriter.reindexDocuments(keyFieldName,query); |
| 426 } | |
| 427 | |
| 1486 | 428 private void writeOp(int op) throws IOException { |
| 1465 | 429 log.writeLong(System.currentTimeMillis()); |
| 430 log.writeByte(op); | |
| 431 } | |
| 432 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
433 // return whether stopped at tag |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
434 public synchronized boolean playLogs(String upToTag) throws IOException { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
435 return playLogs( logReaders(logs), indexWriter, upToTag ); |
| 1465 | 436 } |
| 437 | |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
438 private static List<LogInputStream> logReaders(List<LogFile> logs) throws IOException { |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
439 List<LogInputStream> logReaders = new ArrayList<LogInputStream>(); |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
440 for( LogFile log : logs ) { |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
441 logReaders.add( log.input() ); |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
442 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
443 return logReaders; |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
444 } |
|
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
445 |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
446 private static boolean playLogs(List<LogInputStream> logReaders,LuceneIndexWriter indexWriter,String upToTag) |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
447 throws IOException |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
448 { |
| 1465 | 449 if( numDocs(indexWriter) != 0 ) |
| 450 throw new RuntimeException ("not empty"); | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
451 boolean rtn = false; |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
452 for( LogInputStream reader : logReaders ) { |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
453 if( playLog(reader,indexWriter,upToTag) ) { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
454 rtn = true; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
455 break; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
456 } |
| 1465 | 457 } |
| 458 indexWriter.commit(); | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
459 return rtn; |
| 1465 | 460 } |
| 461 | |
| 462 private static int numDocs(LuceneIndexWriter indexWriter) throws IOException { | |
| 463 IndexReader reader = indexWriter.openReader(); | |
| 464 int n = reader.numDocs(); | |
| 465 reader.close(); | |
| 466 return n; | |
| 467 } | |
| 468 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
469 private static boolean playLog(LogInputStream in,LuceneIndexWriter indexWriter,String upToTag) |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
470 throws IOException |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
471 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
472 boolean rtn = false; |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
473 while( in.available() > 0 ) { |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
474 if( playOp(in,indexWriter,upToTag) ) { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
475 rtn = true; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
476 break; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
477 } |
| 1465 | 478 } |
|
1484
1fa6e8ec2d53
lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
479 in.close(); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
480 return rtn; |
| 1465 | 481 } |
| 482 | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
483 private static boolean playOp(LogInputStream in,LuceneIndexWriter indexWriter,String upToTag) throws IOException { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
484 in.readLong(); // time |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
485 int op = in.readByte(); |
| 1461 | 486 switch(op) { |
| 487 case OP_DELETE_ALL: | |
| 488 indexWriter.deleteAll(); | |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
489 return false; |
| 1461 | 490 case OP_DELETE_DOCUMENTS: |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
491 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
492 Query query = in.readQuery(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
493 //System.out.println("OP_DELETE_DOCUMENTS "+query); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
494 indexWriter.deleteDocuments(query); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
495 return false; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
496 } |
| 1461 | 497 case OP_ADD_DOCUMENT: |
| 1465 | 498 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
499 Map storedFields = in.readMap(); |
| 1465 | 500 indexWriter.addDocument(storedFields); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
501 return false; |
| 1465 | 502 } |
| 1461 | 503 case OP_UPDATE_DOCUMENT: |
| 1465 | 504 { |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
505 String keyFieldName = in.readUTF(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1475
diff
changeset
|
506 Map storedFields = in.readMap(); |
| 1465 | 507 indexWriter.updateDocument(keyFieldName,storedFields); |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
508 return false; |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
509 } |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
510 case OP_TAG: |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
511 { |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
512 String tag = in.readUTF(); |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
513 return tag.equals(upToTag); |
| 1465 | 514 } |
| 1461 | 515 default: |
| 516 throw new RuntimeException("invalid op "+op); | |
| 517 } | |
| 518 } | |
| 519 | |
| 1465 | 520 private static void dump(LuceneIndexWriter indexWriter) throws IOException { |
| 521 IndexReader reader = indexWriter.openReader(); | |
| 522 IndexSearcher searcher = new IndexSearcher(reader); | |
| 523 Query query = new MatchAllDocsQuery(); | |
| 524 TopDocs td = searcher.search(query,100); | |
| 525 System.out.println("totalHits = "+td.totalHits); | |
| 526 for( int i=0; i<td.scoreDocs.length; i++ ) { | |
| 527 Document doc = searcher.doc(td.scoreDocs[i].doc); | |
| 528 System.out.println(LuceneUtils.toMap(doc)); | |
| 1461 | 529 } |
| 1465 | 530 System.out.println(); |
| 531 reader.close(); | |
| 1461 | 532 } |
| 1465 | 533 |
| 1461 | 534 } |