Mercurial Hosting > luan

comparison src/goodjava/lucene/logging/LoggingIndexWriter.java @ 1677:ea7075b7afe1

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
switch to index.json
author Franklin Schmidt <fschmidt@gmail.com>
date Tue, 31 May 2022 14:36:16 -0600
parents 52241b69c339
children f54484aae295
comparison
equal deleted inserted replaced
1676:daa4214fa91a 1677:ea7075b7afe1
1 package goodjava.lucene.logging; 1 package goodjava.lucene.logging;
2 2
3 import java.io.File; 3 import java.io.File;
4 import java.io.RandomAccessFile; 4 import java.io.RandomAccessFile;
5 import java.io.ByteArrayOutputStream; 5 import java.io.ByteArrayOutputStream;
6 import java.io.DataOutputStream; 6 import java.io.OutputStreamWriter;
7 import java.io.DataInputStream; 7 import java.io.FileReader;
8 import java.io.FileInputStream; 8 import java.io.Writer;
9 import java.io.IOException; 9 import java.io.IOException;
10 import java.util.List;
11 import java.util.ArrayList;
10 import java.util.Map; 12 import java.util.Map;
11 import java.util.Set; 13 import java.util.Set;
12 import java.util.HashSet; 14 import java.util.HashSet;
15 import java.util.LinkedHashMap;
13 import java.util.Random; 16 import java.util.Random;
14 import java.util.concurrent.TimeUnit; 17 import java.util.concurrent.TimeUnit;
15 import org.apache.lucene.document.Document; 18 import org.apache.lucene.document.Document;
16 import org.apache.lucene.index.DirectoryReader; 19 import org.apache.lucene.index.DirectoryReader;
17 import org.apache.lucene.index.IndexReader; 20 import org.apache.lucene.index.IndexReader;
25 import org.apache.lucene.search.SortField; 28 import org.apache.lucene.search.SortField;
26 import org.apache.lucene.search.Sort; 29 import org.apache.lucene.search.Sort;
27 import org.apache.lucene.store.Directory; 30 import org.apache.lucene.store.Directory;
28 import org.apache.lucene.store.FSDirectory; 31 import org.apache.lucene.store.FSDirectory;
29 import goodjava.io.IoUtils; 32 import goodjava.io.IoUtils;
33 import goodjava.json.JsonParser;
34 import goodjava.json.JsonToString;
35 import goodjava.parser.ParseException;
30 import goodjava.lucene.api.GoodIndexWriter; 36 import goodjava.lucene.api.GoodIndexWriter;
31 import goodjava.lucene.api.LuceneIndexWriter; 37 import goodjava.lucene.api.LuceneIndexWriter;
32 import goodjava.lucene.api.GoodCollector; 38 import goodjava.lucene.api.GoodCollector;
33 import goodjava.lucene.api.LuceneUtils; 39 import goodjava.lucene.api.LuceneUtils;
34 import goodjava.logging.Logger; 40 import goodjava.logging.Logger;
49 public boolean wasCreated; 55 public boolean wasCreated;
50 private final File logDir; 56 private final File logDir;
51 private final long logTime; 57 private final long logTime;
52 protected final LogFile[] logs = new LogFile[3]; 58 protected final LogFile[] logs = new LogFile[3];
53 private LogOutputStream log; 59 private LogOutputStream log;
54 private final File index; 60 private final File indexFile;
55 private final SemaphoreLock mergeLock = new SemaphoreLock(); 61 private final SemaphoreLock mergeLock = new SemaphoreLock();
56 62
57 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime) 63 public LoggingIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime)
58 throws IOException 64 throws IOException
59 { 65 {
61 this.logDir = logDir; 67 this.logDir = logDir;
62 this.logTime = logTime; 68 this.logTime = logTime;
63 IoUtils.mkdirs(logDir); 69 IoUtils.mkdirs(logDir);
64 if( !logDir.isDirectory() ) 70 if( !logDir.isDirectory() )
65 throw new RuntimeException(); 71 throw new RuntimeException();
66 index = new File(logDir,"index"); 72 indexFile = new File(logDir,"index.json");
67 if( index.exists() ) { 73 if( indexFile.exists() ) {
68 DataInputStream dis = new DataInputStream(new FileInputStream(index));
69 try { 74 try {
70 if( dis.readInt() == version ) { 75 Map map = (Map)JsonParser.parse( IoUtils.readAll(new FileReader(indexFile)) );
76 if( (Integer)map.get("version") == version ) {
77 List fileNames = (List)map.get("files");
71 for( int i=0; i<logs.length; i++ ) { 78 for( int i=0; i<logs.length; i++ ) {
72 File file = new File( logDir, dis.readUTF() ); 79 File file = new File( logDir, (String)fileNames.get(i) );
73 logs[i] = new LogFile(file); 80 logs[i] = new LogFile(file);
74 } 81 }
75 deleteUnusedFiles(); 82 deleteUnusedFiles();
76 setLog(); 83 setLog();
77 wasCreated = false; 84 wasCreated = false;
78 return; 85 return;
79 } 86 }
80 } finally { 87 } catch(ParseException e) {
81 dis.close(); 88 logger.error("bad index.json",e);
82 } 89 }
83 } 90 }
84 logger.info("building new logs"); 91 logger.info("building new logs");
85 for( int i=0; i<logs.length; i++ ) { 92 for( int i=0; i<logs.length; i++ ) {
86 logs[i] = newLogFile(); 93 logs[i] = newLogFile();
155 } while( file.exists() ); 162 } while( file.exists() );
156 return new LogFile(file); 163 return new LogFile(file);
157 } 164 }
158 165
159 private void deleteUnusedFiles() throws IOException { 166 private void deleteUnusedFiles() throws IOException {
160 deleteUnusedFiles(logs,index); 167 deleteUnusedFiles(logs,indexFile);
161 } 168 }
162 169
163 private static void deleteUnusedFiles(LogFile[] logs,File index) throws IOException { 170 private static void deleteUnusedFiles(LogFile[] logs,File indexFile) throws IOException {
164 Set<String> used = new HashSet<String>(); 171 Set<String> used = new HashSet<String>();
165 used.add( index.getName() ); 172 used.add( indexFile.getName() );
166 for( LogFile lf : logs ) { 173 for( LogFile lf : logs ) {
167 used.add( lf.file.getName() ); 174 used.add( lf.file.getName() );
168 } 175 }
169 for( File f : index.getParentFile().listFiles() ) { 176 for( File f : indexFile.getParentFile().listFiles() ) {
170 if( !used.contains(f.getName()) ) { 177 if( !used.contains(f.getName()) ) {
171 IoUtils.deleteRecursively(f); 178 IoUtils.deleteRecursively(f);
172 } 179 }
173 } 180 }
174 } 181 }
175 182
176 private void writeIndex() throws IOException { 183 private void writeIndex() throws IOException {
177 writeIndex(logs,index); 184 writeIndex(logs,indexFile);
178 } 185 }
179 186
180 public static void writeIndex(LogFile[] logs,File index) throws IOException { 187 public static void writeIndex(LogFile[] logs,File indexFile) throws IOException {
181 if( logs.length != 3 ) 188 if( logs.length != 3 )
182 throw new RuntimeException(); 189 throw new RuntimeException();
183 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 190 Map map = new LinkedHashMap();
184 DataOutputStream dos = new DataOutputStream(baos); 191 map.put("version",version);
185 dos.writeInt(version); 192 List fileNames = new ArrayList();
186 for( LogFile lf : logs ) { 193 for( LogFile lf : logs ) {
187 String fileName = lf.file.getName(); 194 String fileName = lf.file.getName();
188 dos.writeUTF(fileName); 195 fileNames.add(fileName);
189 } 196 }
190 dos.close(); 197 map.put("files",fileNames);
191 RandomAccessFile raf = new RandomAccessFile( index, "rwd" ); 198 ByteArrayOutputStream baos = new ByteArrayOutputStream();
199 Writer writer = new OutputStreamWriter(baos);
200 writer.write( new JsonToString().toString(map) );
201 writer.write( '\n' );
202 writer.close();
203 RandomAccessFile raf = new RandomAccessFile( indexFile, "rwd" );
192 raf.write( baos.toByteArray() ); 204 raf.write( baos.toByteArray() );
193 raf.close(); 205 raf.close();
194 deleteUnusedFiles(logs,index); 206 deleteUnusedFiles(logs,indexFile);
195 //logger.info("writeIndex "+logs.toString()); 207 //logger.info("writeIndex "+logs.toString());
196 } 208 }
197 209
198 private void mergeLogs() throws IOException { 210 private void mergeLogs() throws IOException {
199 logger.info("merge"); 211 logger.info("merge");