Mercurial Hosting > luan
comparison src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1677:ea7075b7afe1
switch to index.json
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Tue, 31 May 2022 14:36:16 -0600 |
parents | 52241b69c339 |
children | f54484aae295 |
comparison
equal
deleted
inserted
replaced
1676:daa4214fa91a | 1677:ea7075b7afe1 |
---|---|
1 package goodjava.lucene.logging; | 1 package goodjava.lucene.logging; |
2 | 2 |
3 import java.io.File; | 3 import java.io.File; |
4 import java.io.RandomAccessFile; | 4 import java.io.RandomAccessFile; |
5 import java.io.ByteArrayOutputStream; | 5 import java.io.ByteArrayOutputStream; |
6 import java.io.DataOutputStream; | 6 import java.io.OutputStreamWriter; |
7 import java.io.DataInputStream; | 7 import java.io.FileReader; |
8 import java.io.FileInputStream; | 8 import java.io.Writer; |
9 import java.io.IOException; | 9 import java.io.IOException; |
10 import java.util.List; | |
11 import java.util.ArrayList; | |
10 import java.util.Map; | 12 import java.util.Map; |
11 import java.util.Set; | 13 import java.util.Set; |
12 import java.util.HashSet; | 14 import java.util.HashSet; |
15 import java.util.LinkedHashMap; | |
13 import java.util.Random; | 16 import java.util.Random; |
14 import java.util.concurrent.TimeUnit; | 17 import java.util.concurrent.TimeUnit; |
15 import org.apache.lucene.document.Document; | 18 import org.apache.lucene.document.Document; |
16 import org.apache.lucene.index.DirectoryReader; | 19 import org.apache.lucene.index.DirectoryReader; |
17 import org.apache.lucene.index.IndexReader; | 20 import org.apache.lucene.index.IndexReader; |
25 import org.apache.lucene.search.SortField; | 28 import org.apache.lucene.search.SortField; |
26 import org.apache.lucene.search.Sort; | 29 import org.apache.lucene.search.Sort; |
27 import org.apache.lucene.store.Directory; | 30 import org.apache.lucene.store.Directory; |
28 import org.apache.lucene.store.FSDirectory; | 31 import org.apache.lucene.store.FSDirectory; |
29 import goodjava.io.IoUtils; | 32 import goodjava.io.IoUtils; |
33 import goodjava.json.JsonParser; | |
34 import goodjava.json.JsonToString; | |
35 import goodjava.parser.ParseException; | |
30 import goodjava.lucene.api.GoodIndexWriter; | 36 import goodjava.lucene.api.GoodIndexWriter; |
31 import goodjava.lucene.api.LuceneIndexWriter; | 37 import goodjava.lucene.api.LuceneIndexWriter; |
32 import goodjava.lucene.api.GoodCollector; | 38 import goodjava.lucene.api.GoodCollector; |
33 import goodjava.lucene.api.LuceneUtils; | 39 import goodjava.lucene.api.LuceneUtils; |
34 import goodjava.logging.Logger; | 40 import goodjava.logging.Logger; |
49 public boolean wasCreated; | 55 public boolean wasCreated; |
50 private final File logDir; | 56 private final File logDir; |
51 private final long logTime; | 57 private final long logTime; |
52 protected final LogFile[] logs = new LogFile[3]; | 58 protected final LogFile[] logs = new LogFile[3]; |
53 private LogOutputStream log; | 59 private LogOutputStream log; |
54 private final File index; | 60 private final File indexFile; |
55 private final SemaphoreLock mergeLock = new SemaphoreLock(); | 61 private final SemaphoreLock mergeLock = new SemaphoreLock(); |
56 | 62 |
57 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime) | 63 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime) |
58 throws IOException | 64 throws IOException |
59 { | 65 { |
61 this.logDir = logDir; | 67 this.logDir = logDir; |
62 this.logTime = logTime; | 68 this.logTime = logTime; |
63 IoUtils.mkdirs(logDir); | 69 IoUtils.mkdirs(logDir); |
64 if( !logDir.isDirectory() ) | 70 if( !logDir.isDirectory() ) |
65 throw new RuntimeException(); | 71 throw new RuntimeException(); |
66 index = new File(logDir,"index"); | 72 indexFile = new File(logDir,"index.json"); |
67 if( index.exists() ) { | 73 if( indexFile.exists() ) { |
68 DataInputStream dis = new DataInputStream(new FileInputStream(index)); | |
69 try { | 74 try { |
70 if( dis.readInt() == version ) { | 75 Map map = (Map)JsonParser.parse( IoUtils.readAll(new FileReader(indexFile)) ); |
76 if( (Integer)map.get("version") == version ) { | |
77 List fileNames = (List)map.get("files"); | |
71 for( int i=0; i<logs.length; i++ ) { | 78 for( int i=0; i<logs.length; i++ ) { |
72 File file = new File( logDir, dis.readUTF() ); | 79 File file = new File( logDir, (String)fileNames.get(i) ); |
73 logs[i] = new LogFile(file); | 80 logs[i] = new LogFile(file); |
74 } | 81 } |
75 deleteUnusedFiles(); | 82 deleteUnusedFiles(); |
76 setLog(); | 83 setLog(); |
77 wasCreated = false; | 84 wasCreated = false; |
78 return; | 85 return; |
79 } | 86 } |
80 } finally { | 87 } catch(ParseException e) { |
81 dis.close(); | 88 logger.error("bad index.json",e); |
82 } | 89 } |
83 } | 90 } |
84 logger.info("building new logs"); | 91 logger.info("building new logs"); |
85 for( int i=0; i<logs.length; i++ ) { | 92 for( int i=0; i<logs.length; i++ ) { |
86 logs[i] = newLogFile(); | 93 logs[i] = newLogFile(); |
155 } while( file.exists() ); | 162 } while( file.exists() ); |
156 return new LogFile(file); | 163 return new LogFile(file); |
157 } | 164 } |
158 | 165 |
159 private void deleteUnusedFiles() throws IOException { | 166 private void deleteUnusedFiles() throws IOException { |
160 deleteUnusedFiles(logs,index); | 167 deleteUnusedFiles(logs,indexFile); |
161 } | 168 } |
162 | 169 |
163 private static void deleteUnusedFiles(LogFile[] logs,File index) throws IOException { | 170 private static void deleteUnusedFiles(LogFile[] logs,File indexFile) throws IOException { |
164 Set<String> used = new HashSet<String>(); | 171 Set<String> used = new HashSet<String>(); |
165 used.add( index.getName() ); | 172 used.add( indexFile.getName() ); |
166 for( LogFile lf : logs ) { | 173 for( LogFile lf : logs ) { |
167 used.add( lf.file.getName() ); | 174 used.add( lf.file.getName() ); |
168 } | 175 } |
169 for( File f : index.getParentFile().listFiles() ) { | 176 for( File f : indexFile.getParentFile().listFiles() ) { |
170 if( !used.contains(f.getName()) ) { | 177 if( !used.contains(f.getName()) ) { |
171 IoUtils.deleteRecursively(f); | 178 IoUtils.deleteRecursively(f); |
172 } | 179 } |
173 } | 180 } |
174 } | 181 } |
175 | 182 |
176 private void writeIndex() throws IOException { | 183 private void writeIndex() throws IOException { |
177 writeIndex(logs,index); | 184 writeIndex(logs,indexFile); |
178 } | 185 } |
179 | 186 |
180 public static void writeIndex(LogFile[] logs,File index) throws IOException { | 187 public static void writeIndex(LogFile[] logs,File indexFile) throws IOException { |
181 if( logs.length != 3 ) | 188 if( logs.length != 3 ) |
182 throw new RuntimeException(); | 189 throw new RuntimeException(); |
183 ByteArrayOutputStream baos = new ByteArrayOutputStream(); | 190 Map map = new LinkedHashMap(); |
184 DataOutputStream dos = new DataOutputStream(baos); | 191 map.put("version",version); |
185 dos.writeInt(version); | 192 List fileNames = new ArrayList(); |
186 for( LogFile lf : logs ) { | 193 for( LogFile lf : logs ) { |
187 String fileName = lf.file.getName(); | 194 String fileName = lf.file.getName(); |
188 dos.writeUTF(fileName); | 195 fileNames.add(fileName); |
189 } | 196 } |
190 dos.close(); | 197 map.put("files",fileNames); |
191 RandomAccessFile raf = new RandomAccessFile( index, "rwd" ); | 198 ByteArrayOutputStream baos = new ByteArrayOutputStream(); |
199 Writer writer = new OutputStreamWriter(baos); | |
200 writer.write( new JsonToString().toString(map) ); | |
201 writer.write( '\n' ); | |
202 writer.close(); | |
203 RandomAccessFile raf = new RandomAccessFile( indexFile, "rwd" ); | |
192 raf.write( baos.toByteArray() ); | 204 raf.write( baos.toByteArray() ); |
193 raf.close(); | 205 raf.close(); |
194 deleteUnusedFiles(logs,index); | 206 deleteUnusedFiles(logs,indexFile); |
195 //logger.info("writeIndex "+logs.toString()); | 207 //logger.info("writeIndex "+logs.toString()); |
196 } | 208 } |
197 | 209 |
198 private void mergeLogs() throws IOException { | 210 private void mergeLogs() throws IOException { |
199 logger.info("merge"); | 211 logger.info("merge"); |