annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1740:c9c974817d0c

add ServerSentEvents.sweep
author Franklin Schmidt <fschmidt@gmail.com>
date Thu, 03 Nov 2022 22:05:11 -0600
parents 39287902fb0c
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
1 package goodjava.lucene.logging;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
2
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
3 import java.io.File;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
4 import java.io.RandomAccessFile;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
5 import java.io.ByteArrayOutputStream;
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
6 import java.io.OutputStreamWriter;
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
7 import java.io.FileReader;
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
8 import java.io.Writer;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
9 import java.io.IOException;
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
10 import java.util.List;
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
11 import java.util.ArrayList;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
12 import java.util.Map;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
13 import java.util.Set;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
14 import java.util.HashSet;
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
15 import java.util.LinkedHashMap;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
16 import java.util.Random;
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
17 import java.util.concurrent.TimeUnit;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
18 import org.apache.lucene.document.Document;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
19 import org.apache.lucene.index.DirectoryReader;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
20 import org.apache.lucene.index.IndexReader;
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
21 import org.apache.lucene.index.IndexWriter;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
22 import org.apache.lucene.index.Term;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
23 import org.apache.lucene.search.IndexSearcher;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
24 import org.apache.lucene.search.Query;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
25 import org.apache.lucene.search.MatchAllDocsQuery;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
26 import org.apache.lucene.search.TopDocs;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
27 import org.apache.lucene.search.PrefixQuery;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
28 import org.apache.lucene.search.SortField;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
29 import org.apache.lucene.search.Sort;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
30 import org.apache.lucene.store.Directory;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
31 import org.apache.lucene.store.FSDirectory;
1473
6c6ce14db6a8 add goodjava.io
Franklin Schmidt <fschmidt@gmail.com>
parents: 1465
diff changeset
32 import goodjava.io.IoUtils;
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
33 import goodjava.json.JsonParser;
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
34 import goodjava.json.JsonToString;
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
35 import goodjava.parser.ParseException;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
36 import goodjava.lucene.api.GoodIndexWriter;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
37 import goodjava.lucene.api.LuceneIndexWriter;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
38 import goodjava.lucene.api.GoodCollector;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
39 import goodjava.lucene.api.LuceneUtils;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
40 import goodjava.logging.Logger;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
41 import goodjava.logging.LoggerFactory;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
42
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
43
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1487
diff changeset
44 public class LoggingIndexWriter implements GoodIndexWriter {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
45 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class);
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
46 private static final int version = 2;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
47 private static final int OP_DELETE_ALL = 1;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
48 private static final int OP_DELETE_DOCUMENTS = 2;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
49 private static final int OP_ADD_DOCUMENT = 3;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
50 private static final int OP_UPDATE_DOCUMENT = 4;
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
51 private static final int OP_TAG = 5;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
52 private static final Random rnd = new Random();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
53
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
54 public final LuceneIndexWriter indexWriter;
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
55 public boolean wasCreated;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
56 private final File logDir;
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
57 private final long logTime;
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
58 protected final LogFile[] logs = new LogFile[3];
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
59 private LogOutputStream log;
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
60 private final File indexFile;
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
61 private final SemaphoreLock mergeLock = new SemaphoreLock();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
62
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
63 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime)
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
64 throws IOException
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
65 {
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
66 this.indexWriter = indexWriter;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
67 this.logDir = logDir;
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
68 this.logTime = logTime;
1501
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
69 IoUtils.mkdirs(logDir);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
70 if( !logDir.isDirectory() )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
71 throw new RuntimeException();
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
72 indexFile = new File(logDir,"index.json");
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
73 if( indexFile.exists() ) {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
74 try {
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
75 Map map = (Map)JsonParser.parse( IoUtils.readAll(new FileReader(indexFile)) );
1679
Franklin Schmidt <fschmidt@gmail.com>
parents: 1678
diff changeset
76 if( ((Number)map.get("version")).intValue() == version ) {
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
77 List fileNames = (List)map.get("files");
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
78 for( int i=0; i<logs.length; i++ ) {
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
79 File file = new File( logDir, (String)fileNames.get(i) );
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
80 logs[i] = new LogFile(file);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
81 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
82 deleteUnusedFiles();
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
83 setLog();
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
84 wasCreated = false;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
85 return;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
86 }
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
87 } catch(ParseException e) {
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
88 logger.error("bad index.json",e);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
89 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
90 }
1556
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
91 logger.info("building new logs");
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
92 for( int i=0; i<logs.length; i++ ) {
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
93 logs[i] = newLogFile();
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
94 }
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
95 LogOutputStream log = logs[0].output();
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
96 logLucene( System.currentTimeMillis(), log, indexWriter );
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
97 log.close();
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
98 writeIndex();
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
99 setLog();
52241b69c339 lucene logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1551
diff changeset
100 logger.info("done building new logs");
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
101 wasCreated = true;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
102 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
103
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
104 public IndexReader openReader() throws IOException {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
105 return indexWriter.openReader();
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
106 }
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
107
1539
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
108 public IndexWriter getLuceneIndexWriter() {
c27dc6af87ca remove nextId
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
109 return indexWriter.getLuceneIndexWriter();
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
110 }
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
111
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
112 private void setLog() throws IOException {
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
113 if( log != null )
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
114 log.close();
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
115 log = logs[2].output();
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
116 }
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
117 /*
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
118 public synchronized boolean isMerging() {
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
119 return mergeLock.isLocked();
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
120 }
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
121 */
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
122 private void getMergeLock() {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
123 try {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
124 if( !mergeLock.tryLock(1,TimeUnit.MINUTES) )
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
125 throw new RuntimeException("failed to acquire lock");
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
126 } catch(InterruptedException e) {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
127 throw new RuntimeException(e);
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
128 }
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
129 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
130
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
131 public synchronized void logLucene()
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
132 throws IOException
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
133 {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
134 logLucene( System.currentTimeMillis(), log, indexWriter );
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
135 }
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
136
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
137 private static void logLucene(long time,LogOutputStream log,LuceneIndexWriter indexWriter)
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
138 throws IOException
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
139 {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
140 log.writeLong(time);
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
141 log.writeByte(OP_DELETE_ALL);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
142 IndexReader reader = indexWriter.openReader();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
143 final IndexSearcher searcher = new IndexSearcher(reader);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
144 Query query = new MatchAllDocsQuery();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
145 searcher.search( query, new GoodCollector(){
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
146 public void collectDoc(int iDoc) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
147 Document doc = searcher.doc(iDoc);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
148 Map<String,Object> storedFields = LuceneUtils.toMap(doc);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
149 log.writeLong(time);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
150 log.writeByte(OP_ADD_DOCUMENT);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
151 log.writeMap(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
152 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
153 });
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
154 reader.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
155 log.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
156 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
157
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
158 private LogFile newLogFile() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
159 File file;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
160 do {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
161 file = new File(logDir,"_"+rnd.nextInt(100)+".log");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
162 } while( file.exists() );
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
163 return new LogFile(file);
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
164 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
165
1473
6c6ce14db6a8 add goodjava.io
Franklin Schmidt <fschmidt@gmail.com>
parents: 1465
diff changeset
166 private void deleteUnusedFiles() throws IOException {
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
167 deleteUnusedFiles(logs,indexFile);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
168 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
169
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
170 private static void deleteUnusedFiles(LogFile[] logs,File indexFile) throws IOException {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
171 Set<String> used = new HashSet<String>();
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
172 used.add( indexFile.getName() );
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
173 for( LogFile lf : logs ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
174 used.add( lf.file.getName() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
175 }
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
176 for( File f : indexFile.getParentFile().listFiles() ) {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
177 if( !used.contains(f.getName()) ) {
1475
c7b86342857f more IoUtils
Franklin Schmidt <fschmidt@gmail.com>
parents: 1473
diff changeset
178 IoUtils.deleteRecursively(f);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
179 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
180 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
181 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
182
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
183 private void writeIndex() throws IOException {
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
184 writeIndex(logs,indexFile);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
185 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
186
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
187 public static void writeIndex(LogFile[] logs,File indexFile) throws IOException {
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
188 if( logs.length != 3 )
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
189 throw new RuntimeException();
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
190 Map map = new LinkedHashMap();
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
191 map.put("version",version);
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
192 List fileNames = new ArrayList();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
193 for( LogFile lf : logs ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
194 String fileName = lf.file.getName();
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
195 fileNames.add(fileName);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
196 }
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
197 map.put("files",fileNames);
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
198 ByteArrayOutputStream baos = new ByteArrayOutputStream();
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
199 Writer writer = new OutputStreamWriter(baos);
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
200 writer.write( new JsonToString().toString(map) );
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
201 writer.write( '\n' );
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
202 writer.close();
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
203 RandomAccessFile raf = new RandomAccessFile( indexFile, "rwd" );
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
204 raf.write( baos.toByteArray() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
205 raf.close();
1677
ea7075b7afe1 switch to index.json
Franklin Schmidt <fschmidt@gmail.com>
parents: 1556
diff changeset
206 deleteUnusedFiles(logs,indexFile);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
207 //logger.info("writeIndex "+logs.toString());
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
208 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
209
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
210 private void mergeLogs() throws IOException {
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
211 logger.info("merge");
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
212 if( !mergeLock.isLocked() ) {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
213 logger.error("merge without lock");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
214 return;
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
215 }
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
216 LogFile first = logs[0];
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
217 LogFile second = logs[1];
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
218 long lastTime = second.file.lastModified();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
219 File dirFile = new File(logDir,"merge");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
220 if( dirFile.exists() )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
221 throw new RuntimeException();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
222 Directory dir = FSDirectory.open(dirFile);
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
223 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig );
1551
9cc4cee39b8b add LuanOpDoer
Franklin Schmidt <fschmidt@gmail.com>
parents: 1549
diff changeset
224 OpDoer opDoer = new BasicOpDoer(mergeWriter);
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
225 playLog( first.input(), opDoer );
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
226 playLog( second.input(), opDoer );
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
227 mergeWriter.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
228 LogFile merge = newLogFile();
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
229 LogOutputStream log = merge.output();
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
230 logLucene( lastTime, log, mergeWriter );
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
231 log.close();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
232 mergeWriter.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
233 synchronized(this) {
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
234 //check();
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
235 logs[0] = merge;
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
236 logs[1] = logs[2];
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
237 logs[2] = newLogFile();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
238 writeIndex();
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
239 setLog();
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
240 //check(null);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
241 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
242 }
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
243 private final Runnable mergeLogs = new Runnable() { public void run() {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
244 try {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
245 mergeLogs();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
246 } catch(IOException e) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
247 throw new RuntimeException(e);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
248 } finally {
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
249 mergeLock.unlock();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
250 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
251 } };
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
252
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
253 private static class DocIter {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
254 final IndexReader reader;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
255 final TopDocs td;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
256 final int n;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
257 int i = 0;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
258
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
259 DocIter(IndexReader reader,Query query,Sort sort) throws IOException {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
260 this.reader = reader;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
261 IndexSearcher searcher = new IndexSearcher(reader);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
262 this.td = searcher.search(query,10000000,sort);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
263 this.n = td.scoreDocs.length;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
264 if( td.totalHits != n )
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
265 throw new RuntimeException();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
266 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
267
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
268 Document next() throws IOException {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
269 return i < n ? reader.document(td.scoreDocs[i++].doc) : null;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
270 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
271 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
272
1487
Franklin Schmidt <fschmidt@gmail.com>
parents: 1486
diff changeset
273 private volatile boolean isChecking = false;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1486
diff changeset
274
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
275 public boolean check(SortField sortField) throws IOException {
1487
Franklin Schmidt <fschmidt@gmail.com>
parents: 1486
diff changeset
276 if( isChecking )
Franklin Schmidt <fschmidt@gmail.com>
parents: 1486
diff changeset
277 throw new RuntimeException("another check is running");
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
278 isChecking = true;
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
279 try {
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
280 return doCheck(sortField);
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
281 } finally {
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
282 isChecking = false;
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
283 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
284 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1504
diff changeset
285
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
286 protected boolean doCheck(SortField sortField) throws IOException {
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
287 boolean ok = true;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
288 IndexReader indexReader;
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
289 LogInputStream[] logReaders;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
290 synchronized(this) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
291 indexReader = indexWriter.openReader();
1484
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
292 logReaders = logReaders(logs);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
293 }
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
294 try {
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
295 //logger.info("check start");
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
296 indexWriter.check();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
297 File dirFile = new File(logDir,"check");
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
298 IoUtils.deleteRecursively(dirFile);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
299 Directory dir = FSDirectory.open(dirFile);
1528
3bd4d7963456 use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
300 LuceneIndexWriter checkWriter = new LuceneIndexWriter( dir, indexWriter.goodConfig );
1551
9cc4cee39b8b add LuanOpDoer
Franklin Schmidt <fschmidt@gmail.com>
parents: 1549
diff changeset
301 playLogs(logReaders,new BasicOpDoer(checkWriter));
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
302 //logger.info("check lucene");
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
303 IndexReader checkReader = checkWriter.openReader();
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
304 int nCheck = checkReader.numDocs();
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
305 int nOrig = indexReader.numDocs();
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
306 if( nCheck != nOrig ) {
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
307 logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck);
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
308 ok = false;
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
309 }
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
310 if( sortField == null ) {
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
311 if( ok && hash(indexReader) != hash(checkReader) ) {
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
312 logger.error("hash mismatch");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
313 ok = false;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
314 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
315 } else {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
316 Sort sort = new Sort(sortField);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
317 String sortFieldName = sortField.getField();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
318 Query query = new PrefixQuery(new Term(sortFieldName));
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
319 DocIter origIter = new DocIter(indexReader,query,sort);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
320 DocIter checkIter = new DocIter(checkReader,query,sort);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
321 Map<String,Object> origFields = LuceneUtils.toMap(origIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
322 Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
323 while( origFields!=null && checkFields!=null ) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
324 Comparable origFld = (Comparable)origFields.get(sortFieldName);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
325 Comparable checkFld = (Comparable)checkFields.get(sortFieldName);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
326 int cmp = origFld.compareTo(checkFld);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
327 if( cmp==0 ) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
328 if( !origFields.equals(checkFields) ) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
329 logger.error(sortFieldName+" "+origFld+" not equal");
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
330 logger.error("lucene = "+origFields);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
331 logger.error("logs = "+checkFields);
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
332 ok = false;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
333 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
334 origFields = LuceneUtils.toMap(origIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
335 checkFields = LuceneUtils.toMap(checkIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
336 } else if( cmp < 0 ) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
337 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
338 ok = false;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
339 origFields = LuceneUtils.toMap(origIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
340 } else { // >
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
341 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
342 ok = false;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
343 checkFields = LuceneUtils.toMap(checkIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
344 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
345 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
346 while( origFields!=null ) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
347 Comparable origFld = (Comparable)origFields.get(sortFieldName);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
348 logger.error(sortFieldName+" "+origFld+" found in lucene but not logs");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
349 ok = false;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
350 origFields = LuceneUtils.toMap(origIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
351 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
352 while( checkFields!=null ) {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
353 Comparable checkFld = (Comparable)checkFields.get(sortFieldName);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
354 logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene");
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
355 ok = false;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
356 checkFields = LuceneUtils.toMap(checkIter.next());
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
357 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
358 //logger.info("check done");
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
359 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
360 checkReader.close();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
361 checkWriter.close();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
362 IoUtils.deleteRecursively(dirFile);
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
363 //logger.info("check done");
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
364 } finally {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
365 indexReader.close();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
366 }
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
367 return ok;
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
368 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
369
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
370 private static abstract class HashCollector extends GoodCollector {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
371 int total = 0;
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
372 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
373
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
374 private static int hash(IndexReader reader) throws IOException {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
375 final IndexSearcher searcher = new IndexSearcher(reader);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
376 Query query = new MatchAllDocsQuery();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
377 HashCollector col = new HashCollector() {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
378 public void collectDoc(int iDoc) throws IOException {
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
379 Document doc = searcher.doc(iDoc);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
380 Map<String,Object> storedFields = LuceneUtils.toMap(doc);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
381 total += storedFields.hashCode();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
382 }
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
383 };
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
384 searcher.search(query,col);
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
385 return col.total;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
386 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
387
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
388 public synchronized void close() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
389 indexWriter.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
390 log.commit();
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
391 log.close();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
392 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
393
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
394 public synchronized void commit() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
395 indexWriter.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
396 log.commit();
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
397 if( mergeLock.isLocked() )
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
398 return;
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
399 if( logs[1].file.lastModified() < System.currentTimeMillis() - logTime ) {
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1528
diff changeset
400 getMergeLock();
1504
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
401 new Thread(mergeLogs).start();
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
402 // mergeLogs.run();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
403 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
404 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
405
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
406 public synchronized void rollback() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
407 indexWriter.rollback();
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
408 log.rollback();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
409 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
410
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
411 public synchronized void deleteAll() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
412 indexWriter.deleteAll();
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
413 writeOp(OP_DELETE_ALL);
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
414 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
415
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
416 public synchronized void deleteDocuments(Query query) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
417 indexWriter.deleteDocuments(query);
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
418 writeOp(OP_DELETE_DOCUMENTS);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
419 log.writeQuery(query);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
420 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
421
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
422 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
423 indexWriter.addDocument(storedFields);
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
424 writeOp(OP_ADD_DOCUMENT);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
425 log.writeMap(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
426 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
427
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
428 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
429 indexWriter.updateDocument(keyFieldName,storedFields);
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
430 writeOp(OP_UPDATE_DOCUMENT);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
431 log.writeUTF(keyFieldName);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
432 log.writeMap(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
433 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
434
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
435 public synchronized void tag(String tag) throws IOException {
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
436 writeOp(OP_TAG);
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
437 log.writeUTF(tag);
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
438 }
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
439
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
440 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException {
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
441 indexWriter.reindexDocuments(keyFieldName,query);
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
442 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
443
1486
2469aa31f31b LogOutputStream
Franklin Schmidt <fschmidt@gmail.com>
parents: 1484
diff changeset
444 private void writeOp(int op) throws IOException {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
445 log.writeLong(System.currentTimeMillis());
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
446 log.writeByte(op);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
447 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
448
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
449 public synchronized void playLogs(OpDoer opDoer) throws IOException {
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
450 if( opDoer == null )
1551
9cc4cee39b8b add LuanOpDoer
Franklin Schmidt <fschmidt@gmail.com>
parents: 1549
diff changeset
451 opDoer = new BasicOpDoer(indexWriter);
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
452 playLogs( logReaders(logs), opDoer );
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
453 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
454
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
455 private static LogInputStream[] logReaders(LogFile[] logs) throws IOException {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
456 LogInputStream[] logReaders = new LogInputStream[logs.length];
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
457 for( int i=0; i<logs.length; i++ ) {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
458 logReaders[i] = logs[i].input();
1484
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
459 }
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
460 return logReaders;
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
461 }
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
462
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
463 private static void playLogs(LogInputStream[] logReaders,OpDoer opDoer)
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
464 throws IOException
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
465 {
1484
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
466 for( LogInputStream reader : logReaders ) {
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
467 playLog(reader,opDoer);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
468 }
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
469 opDoer.commit();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
470 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
471
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
472 private static void playLog(LogInputStream in,OpDoer opDoer)
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
473 throws IOException
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
474 {
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
475 while( in.available() > 0 ) {
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
476 playOp(in,opDoer);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
477 }
1484
1fa6e8ec2d53 lucene.logging cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1476
diff changeset
478 in.close();
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
479 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
480
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
481 private static void playOp(LogInputStream in,OpDoer opDoer)
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
482 throws IOException
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
483 {
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
484 long time = in.readLong(); // time
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
485 int op = in.readByte();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
486 switch(op) {
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
487 case OP_DELETE_ALL:
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
488 opDoer.deleteAll(time);
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
489 return;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
490 case OP_DELETE_DOCUMENTS:
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
491 {
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
492 Query query = in.readQuery();
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
493 //System.out.println("OP_DELETE_DOCUMENTS "+query);
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
494 opDoer.deleteDocuments(time,query);
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
495 return;
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
496 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
497 case OP_ADD_DOCUMENT:
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
498 {
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
499 Map storedFields = in.readMap();
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
500 opDoer.addDocument(time,storedFields);
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
501 return;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
502 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
503 case OP_UPDATE_DOCUMENT:
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
504 {
1476
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
505 String keyFieldName = in.readUTF();
7d145095cc0b lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1475
diff changeset
506 Map storedFields = in.readMap();
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
507 opDoer.updateDocument(time,keyFieldName,storedFields);
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
508 return;
1544
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
509 }
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
510 case OP_TAG:
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
511 {
35601f15ecc3 add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents: 1539
diff changeset
512 String tag = in.readUTF();
1549
41c32da4cbd1 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
513 opDoer.tag(time,tag);
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1544
diff changeset
514 return;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
515 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
516 default:
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
517 throw new RuntimeException("invalid op "+op);
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
518 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
519 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
520
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
521 private static void dump(LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
522 IndexReader reader = indexWriter.openReader();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
523 IndexSearcher searcher = new IndexSearcher(reader);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
524 Query query = new MatchAllDocsQuery();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
525 TopDocs td = searcher.search(query,100);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
526 System.out.println("totalHits = "+td.totalHits);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
527 for( int i=0; i<td.scoreDocs.length; i++ ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
528 Document doc = searcher.doc(td.scoreDocs[i].doc);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
529 System.out.println(LuceneUtils.toMap(doc));
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
530 }
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
531 System.out.println();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
532 reader.close();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
533 }
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
534
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
535 }