annotate src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1474:13cbce740e1e

LowercaseAnalyzer
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 17 Apr 2020 21:53:09 -0600
parents 6c6ce14db6a8
children c7b86342857f
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
1 package goodjava.lucene.logging;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
2
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
3 import java.io.File;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
4 import java.io.RandomAccessFile;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
5 import java.io.ByteArrayOutputStream;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
6 import java.io.DataOutputStream;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
7 import java.io.DataInputStream;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
8 import java.io.FileInputStream;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
9 import java.io.IOException;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
10 import java.util.Map;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
11 import java.util.Set;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
12 import java.util.HashSet;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
13 import java.util.List;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
14 import java.util.ArrayList;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
15 import java.util.Random;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
16 import org.apache.lucene.document.Document;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
17 import org.apache.lucene.index.DirectoryReader;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
18 import org.apache.lucene.index.IndexReader;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
19 import org.apache.lucene.search.IndexSearcher;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
20 import org.apache.lucene.search.Query;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
21 import org.apache.lucene.search.MatchAllDocsQuery;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
22 import org.apache.lucene.search.TopDocs;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
23 import org.apache.lucene.store.Directory;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
24 import org.apache.lucene.store.FSDirectory;
1473
6c6ce14db6a8 add goodjava.io
Franklin Schmidt <fschmidt@gmail.com>
parents: 1465
diff changeset
25 import goodjava.io.IoUtils;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
26 import goodjava.lucene.api.GoodIndexWriter;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
27 import goodjava.lucene.api.LuceneIndexWriter;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
28 import goodjava.lucene.api.GoodCollector;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
29 import goodjava.lucene.api.LuceneUtils;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
30 import goodjava.logging.Logger;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
31 import goodjava.logging.LoggerFactory;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
32
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
33
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
34 public final class LoggingIndexWriter implements GoodIndexWriter {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
35 private static final Logger logger = LoggerFactory.getLogger(LoggingIndexWriter.class);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
36 private static final int version = 1;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
37 private static final int OP_DELETE_ALL = 1;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
38 private static final int OP_DELETE_DOCUMENTS = 2;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
39 private static final int OP_ADD_DOCUMENT = 3;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
40 private static final int OP_UPDATE_DOCUMENT = 4;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
41 private static final Random rnd = new Random();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
42
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
43 public final LuceneIndexWriter indexWriter;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
44 private final File logDir;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
45 private final List<LogFile> logs = new ArrayList<LogFile>();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
46 private final File index;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
47 private boolean isMerging = false;
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
48
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
49 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir) throws IOException {
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
50 this.indexWriter = indexWriter;
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
51 this.logDir = logDir;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
52 logDir.mkdirs();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
53 if( !logDir.isDirectory() )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
54 throw new RuntimeException();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
55 index = new File(logDir,"index");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
56 if( index.exists() ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
57 DataInputStream dis = new DataInputStream(new FileInputStream(index));
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
58 try {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
59 if( dis.readInt() == version ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
60 final int n = dis.readInt();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
61 for( int i=0; i<n; i++ ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
62 File file = new File( logDir, dis.readUTF() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
63 logs.add( new LogFile(file,"rwd") );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
64 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
65 deleteUnusedFiles();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
66 log().gotoEnd();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
67 return;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
68 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
69 } finally {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
70 dis.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
71 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
72 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
73 for( int i=0; i<2; i++ ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
74 logs.add( newLogFile() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
75 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
76 isMerging = true;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
77 new Thread(new Runnable(){public void run(){
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
78 try {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
79 logLucene( System.currentTimeMillis(), logs.get(0), indexWriter );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
80 synchronized(LoggingIndexWriter.this) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
81 writeIndex();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
82 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
83 } catch(IOException e) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
84 throw new RuntimeException(e);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
85 } finally {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
86 synchronized(LoggingIndexWriter.this) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
87 isMerging = false;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
88 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
89 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
90 }}).start();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
91 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
92
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
93 private static void logLucene(long time,LogFile log,LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
94 IndexReader reader = indexWriter.openReader();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
95 final IndexSearcher searcher = new IndexSearcher(reader);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
96 Query query = new MatchAllDocsQuery();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
97 searcher.search( query, new GoodCollector(){
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
98 public void collectDoc(int iDoc) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
99 Document doc = searcher.doc(iDoc);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
100 Map<String,Object> storedFields = LuceneUtils.toMap(doc);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
101 log.writeLong(time);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
102 log.writeByte(OP_ADD_DOCUMENT);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
103 log.writeMap(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
104 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
105 });
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
106 reader.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
107 log.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
108 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
109
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
110 private LogFile newLogFile() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
111 File file;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
112 do {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
113 file = new File(logDir,"_"+rnd.nextInt(100)+".log");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
114 } while( file.exists() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
115 return new LogFile(file,"rwd");
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
116 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
117
1473
6c6ce14db6a8 add goodjava.io
Franklin Schmidt <fschmidt@gmail.com>
parents: 1465
diff changeset
118 private void deleteUnusedFiles() throws IOException {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
119 Set<String> used = new HashSet<String>();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
120 used.add( index.getName() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
121 for( LogFile lf : logs ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
122 used.add( lf.file.getName() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
123 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
124 for( File f : logDir.listFiles() ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
125 if( !used.contains(f.getName()) ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
126 deleteFile(f);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
127 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
128 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
129 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
130
1473
6c6ce14db6a8 add goodjava.io
Franklin Schmidt <fschmidt@gmail.com>
parents: 1465
diff changeset
131 private static void deleteFile(File file) throws IOException {
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
132 if( file.isDirectory() ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
133 for( File f : file.listFiles() ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
134 deleteFile(f);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
135 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
136 }
1473
6c6ce14db6a8 add goodjava.io
Franklin Schmidt <fschmidt@gmail.com>
parents: 1465
diff changeset
137 IoUtils.delete(file);
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
138 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
139
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
140 private void writeIndex() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
141 ByteArrayOutputStream baos = new ByteArrayOutputStream();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
142 DataOutputStream dos = new DataOutputStream(baos);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
143 dos.writeInt(version);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
144 dos.writeInt(logs.size());
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
145 for( LogFile lf : logs ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
146 String fileName = lf.file.getName();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
147 dos.writeUTF(fileName);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
148 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
149 dos.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
150 RandomAccessFile raf = new RandomAccessFile( index, "rwd" );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
151 raf.write( baos.toByteArray() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
152 raf.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
153 deleteUnusedFiles();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
154 logger.info("writeIndex "+logs.toString());
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
155 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
156
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
157 private void mergeLogs() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
158 logger.info("merge");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
159 LogFile first = logs.get(0);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
160 LogFile second = logs.get(1);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
161 second.gotoEnd();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
162 long lastTime = second.readLong();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
163 File dirFile = new File(logDir,"merge");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
164 if( dirFile.exists() )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
165 throw new RuntimeException();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
166 Directory dir = FSDirectory.open(dirFile);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
167 LuceneIndexWriter mergeWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
168 playLog(first,mergeWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
169 playLog(second,mergeWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
170 mergeWriter.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
171 LogFile merge = newLogFile();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
172 logLucene( lastTime, merge, mergeWriter );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
173 mergeWriter.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
174 synchronized(this) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
175 check();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
176 logs.remove(0);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
177 logs.set(0,merge);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
178 writeIndex();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
179 check();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
180 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
181 }
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
182 private final Runnable mergeLogs = new Runnable() { public void run() {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
183 try {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
184 mergeLogs();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
185 /*
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
186 } catch(IOException e) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
187 throw new RuntimeException(e);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
188 */
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
189 } catch(Exception e) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
190 e.printStackTrace();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
191 System.exit(-1);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
192 } finally {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
193 synchronized(LoggingIndexWriter.this) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
194 isMerging = false;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
195 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
196 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
197 } };
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
198
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
199 private void check() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
200 File dirFile = new File(logDir,"check");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
201 if( dirFile.exists() )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
202 throw new RuntimeException();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
203 Directory dir = FSDirectory.open(dirFile);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
204 LuceneIndexWriter checkWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
205 playLog(checkWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
206 int nCheck = numDocs(checkWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
207 int nOrig = numDocs(indexWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
208 if( nCheck != nOrig ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
209 logger.error("nCheck = "+nCheck);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
210 logger.error("nOrig = "+nOrig);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
211 //new Exception().printStackTrace();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
212 Thread.dumpStack();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
213 System.out.println();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
214 System.out.println("indexWriter");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
215 dump(indexWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
216 System.out.println("checkWriter");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
217 dump(checkWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
218 System.exit(-1);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
219 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
220 checkWriter.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
221 deleteFile(dirFile);
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
222 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
223
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
224 private LogFile log() {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
225 return logs.get(logs.size()-1);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
226 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
227
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
228 public synchronized void close() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
229 indexWriter.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
230 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
231 log.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
232 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
233
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
234 public synchronized void commit() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
235 indexWriter.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
236 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
237 log.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
238 if( isMerging )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
239 return;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
240 if( log.length() > logs.get(0).length() ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
241 log.writeLong( System.currentTimeMillis() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
242 logs.add( newLogFile() );
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
243 writeIndex();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
244 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
245 if( logs.size() > 3 ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
246 isMerging = true;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
247 // new Thread(mergeLogs).start();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
248 mergeLogs.run();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
249 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
250 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
251
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
252 public synchronized void rollback() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
253 indexWriter.rollback();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
254 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
255 log.gotoEnd();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
256 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
257
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
258 public synchronized void deleteAll() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
259 indexWriter.deleteAll();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
260 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
261 writeOp(log,OP_DELETE_ALL);
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
262 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
263
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
264 public synchronized void deleteDocuments(Query query) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
265 indexWriter.deleteDocuments(query);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
266 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
267 writeOp(log,OP_DELETE_DOCUMENTS);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
268 log.writeQuery(query);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
269 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
270
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
271 public synchronized void addDocument(Map<String,Object> storedFields) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
272 indexWriter.addDocument(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
273 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
274 writeOp(log,OP_ADD_DOCUMENT);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
275 log.writeMap(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
276 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
277
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
278 public synchronized void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
279 indexWriter.updateDocument(keyFieldName,storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
280 LogFile log = log();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
281 writeOp(log,OP_UPDATE_DOCUMENT);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
282 log.writeUTF(keyFieldName);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
283 log.writeMap(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
284 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
285
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
286 public synchronized void reindexDocuments(String keyFieldName,Query query) throws IOException {
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
287 indexWriter.reindexDocuments(keyFieldName,query);
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
288 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
289
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
290 private void writeOp(LogFile log,int op) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
291 log.writeLong(System.currentTimeMillis());
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
292 log.writeByte(op);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
293 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
294
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
295 public synchronized void playLog() throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
296 playLog(indexWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
297 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
298
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
299 private void playLog(LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
300 if( numDocs(indexWriter) != 0 )
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
301 throw new RuntimeException ("not empty");
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
302 for( LogFile log : logs ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
303 playLog(log,indexWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
304 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
305 indexWriter.commit();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
306 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
307
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
308 private static int numDocs(LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
309 IndexReader reader = indexWriter.openReader();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
310 int n = reader.numDocs();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
311 reader.close();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
312 return n;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
313 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
314
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
315 private static void playLog(LogFile log,LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
316 log.gotoStart();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
317 while( log.hasMore() ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
318 playOp(log,indexWriter);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
319 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
320 }
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
321
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
322 private static void playOp(LogFile log,LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
323 log.readLong(); // time
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
324 int op = log.readByte();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
325 switch(op) {
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
326 case OP_DELETE_ALL:
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
327 indexWriter.deleteAll();
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
328 return;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
329 case OP_DELETE_DOCUMENTS:
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
330 indexWriter.deleteDocuments( log.readQuery() );
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
331 return;
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
332 case OP_ADD_DOCUMENT:
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
333 {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
334 Map storedFields = log.readMap();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
335 indexWriter.addDocument(storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
336 return;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
337 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
338 case OP_UPDATE_DOCUMENT:
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
339 {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
340 String keyFieldName = log.readUTF();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
341 Map storedFields = log.readMap();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
342 indexWriter.updateDocument(keyFieldName,storedFields);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
343 return;
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
344 }
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
345 default:
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
346 throw new RuntimeException("invalid op "+op);
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
347 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
348 }
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
349
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
350 private static void dump(LuceneIndexWriter indexWriter) throws IOException {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
351 IndexReader reader = indexWriter.openReader();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
352 IndexSearcher searcher = new IndexSearcher(reader);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
353 Query query = new MatchAllDocsQuery();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
354 TopDocs td = searcher.search(query,100);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
355 System.out.println("totalHits = "+td.totalHits);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
356 for( int i=0; i<td.scoreDocs.length; i++ ) {
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
357 Document doc = searcher.doc(td.scoreDocs[i].doc);
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
358 System.out.println(LuceneUtils.toMap(doc));
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
359 }
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
360 System.out.println();
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
361 reader.close();
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
362 }
1465
5e3870618377 lucene.logging dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1461
diff changeset
363
1461
e5d48b85351c start lucene.logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
364 }