Mercurial Hosting > luan

changeset 1476:7d145095cc0b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene.logging check
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 19 Apr 2020 20:42:26 -0600
parents c7b86342857f
children 509736ad42e6
files src/goodjava/lucene/api/LuceneIndexWriter.java src/goodjava/lucene/logging/LogFile.java src/goodjava/lucene/logging/LogInputStream.java src/goodjava/lucene/logging/LoggingIndexWriter.java
diffstat 4 files changed, 438 insertions(+), 229 deletions(-) [+]
line wrap: on
line diff
diff -r c7b86342857f -r 7d145095cc0b src/goodjava/lucene/api/LuceneIndexWriter.java
--- a/src/goodjava/lucene/api/LuceneIndexWriter.java	Sat Apr 18 11:02:18 2020 -0600
+++ b/src/goodjava/lucene/api/LuceneIndexWriter.java	Sun Apr 19 20:42:26 2020 -0600
@@ -19,13 +19,17 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.CheckIndex;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Version;
+import goodjava.logging.Logger;
+import goodjava.logging.LoggerFactory;
 
 
 public final class LuceneIndexWriter implements GoodIndexWriter {
+	private static final Logger logger = LoggerFactory.getLogger(LuceneIndexWriter.class);
 	private final FieldAnalyzer fieldAnalyzer = new FieldAnalyzer();
 	public final Version luceneVersion;
 	public final IndexWriterConfig luceneConfig;
@@ -180,4 +184,10 @@
 	public IndexReader openReader() throws IOException {
 		return DirectoryReader.open(luceneWriter.getDirectory());
 	}
+
+	public void check() throws IOException {
+		CheckIndex.Status status = new CheckIndex(luceneWriter.getDirectory()).checkIndex();
+		if( !status.clean )
+			logger.error("index not clean");
+	}
 }
diff -r c7b86342857f -r 7d145095cc0b src/goodjava/lucene/logging/LogFile.java
--- a/src/goodjava/lucene/logging/LogFile.java	Sat Apr 18 11:02:18 2020 -0600
+++ b/src/goodjava/lucene/logging/LogFile.java	Sun Apr 19 20:42:26 2020 -0600
@@ -1,12 +1,15 @@
 package goodjava.lucene.logging;
 
 import java.io.File;
+import java.io.InputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
 import java.io.RandomAccessFile;
+import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.List;
-import java.util.ArrayList;
 import java.util.Map;
-import java.util.LinkedHashMap;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -23,67 +26,97 @@
 import goodjava.logging.LoggerFactory;
 
 
-public class LogFile extends RandomAccessFile {
+public class LogFile {
 	private static final Logger logger = LoggerFactory.getLogger(LogFile.class);
 	public final File file;
+	private final RandomAccessFile raf;
+	private ByteArrayOutputStream baos = new ByteArrayOutputStream();
+	private DataOutput out;
 	private long end;
 
 	public LogFile(File file,String mode) throws IOException {
-		super(file,mode);
 		this.file = file;
+		this.raf = new RandomAccessFile(file,mode);
+		this.out = new DataOutputStream(baos);
 		init();
 	}
 
 	private void init() throws IOException {
-		if( length() == 0 ) {
+		if( raf.length() == 0 ) {
 			end = 8;
-			writeLong(end);
+			raf.writeLong(end);
 		} else {
-			seek(0L);
-			end = readLong();
-			gotoEnd();
+			raf.seek(0L);
+			end = raf.readLong();
+			raf.seek(end);
 		}
 	}
 
+	private LogFile(LogFile lf) throws IOException {
+		this.file = lf.file;
+		this.raf = new RandomAccessFile(file,"r");
+		this.out = null;
+		this.end = lf.end;
+	}
+
+	public LogFile snapshot() throws IOException {
+		return new LogFile(this);
+	}
+
 	public String toString() {
 		return "LogFile<" + file.getName() + ">";
 	}
 
-	public void gotoStart() throws IOException {
-		seek(8L);
+	public long end() {
+		return end;
 	}
 
-	public void gotoEnd() throws IOException {
-		seek(end);
+	public LogInputStream input() throws IOException {
+		byte[] a = new byte[(int)end - 8];
+		raf.seek(8L);
+		raf.readFully(a);
+		return newLogInputStream(new ByteArrayInputStream(a));
+	}
+
+	protected LogInputStream newLogInputStream(InputStream in) {
+		return new LogInputStream(in);
 	}
 
 	public void commit() throws IOException {
-		end = getFilePointer();
-		seek(0L);
-		writeLong(end);
-		gotoEnd();
-	}
-
-	public boolean hasMore() throws IOException {
-		return getFilePointer() < end;
+		raf.seek(end);
+		raf.write(baos.toByteArray());
+		//logger.info("size "+baos.size());
+		if( baos.size() < 10000 ) {
+			baos.reset();
+		} else {
+			baos = new ByteArrayOutputStream();
+			out = new DataOutputStream(baos);
+		}
+		end = raf.getFilePointer();
+		raf.seek(0L);
+		raf.writeLong(end);
 	}
 
-	private static final int TYPE_NULL = 0;
-	private static final int TYPE_STRING = 1;
-	private static final int TYPE_INT = 2;
-	private static final int TYPE_LONG = 3;
-	private static final int TYPE_FLOAT = 4;
-	private static final int TYPE_DOUBLE = 5;
-	private static final int TYPE_BYTES = 6;
-	private static final int TYPE_LIST = 7;
-	private static final int TYPE_QUERY_MATCH_ALL_DOCS = 8;
-	private static final int TYPE_QUERY_TERM = 9;
-	private static final int TYPE_QUERY_PREFIX = 10;
-	private static final int TYPE_QUERY_WILDCARD = 11;
-	private static final int TYPE_QUERY_TERM_RANGE = 12;
-	private static final int TYPE_QUERY_PHRASE = 13;
-	private static final int TYPE_QUERY_NUMERIC_RANGE = 14;
-	private static final int TYPE_QUERY_BOOLEAN = 15;
+	public void rollback() throws IOException {
+		baos.reset();
+	}
+
+	static final int TYPE_NULL = 0;
+	static final int TYPE_STRING = 1;
+	static final int TYPE_INT = 2;
+	static final int TYPE_LONG = 3;
+	static final int TYPE_FLOAT = 4;
+	static final int TYPE_DOUBLE = 5;
+	static final int TYPE_BYTES = 6;
+	static final int TYPE_LIST = 7;
+	static final int TYPE_QUERY_MATCH_ALL_DOCS = 8;
+	static final int TYPE_QUERY_TERM = 9;
+	static final int TYPE_QUERY_PREFIX = 10;
+	static final int TYPE_QUERY_WILDCARD = 11;
+	static final int TYPE_QUERY_TERM_RANGE = 12;
+	static final int TYPE_QUERY_PHRASE = 13;
+	static final int TYPE_QUERY_NUMERIC_RANGE = 14;
+	static final int TYPE_QUERY_BOOLEAN = 15;
 
 	public void writeObject(Object obj) throws IOException {
 		if( obj==null ) {
@@ -195,103 +228,11 @@
 		throw new IllegalArgumentException("invalid type for "+obj);
 	}
 
-	public Object readObject() throws IOException {
-		int type = readByte();
-		return readObject(type);
-	}
-
-	protected Object readObject(int type) throws IOException {
-		switch(type) {
-		case TYPE_NULL:
-			return null;
-		case TYPE_STRING:
-			return readUTF();
-		case TYPE_INT:
-			return readInt();
-		case TYPE_LONG:
-			return readLong();
-		case TYPE_FLOAT:
-			return readFloat();
-		case TYPE_DOUBLE:
-			return readDouble();
-		case TYPE_BYTES:
-			return readByteArray();
-		case TYPE_LIST:
-			return readList();
-		case TYPE_QUERY_MATCH_ALL_DOCS:
-			return new MatchAllDocsQuery();
-		case TYPE_QUERY_TERM:
-			return new TermQuery( readTerm() );
-		case TYPE_QUERY_PREFIX:
-			return new PrefixQuery( readTerm() );
-		case TYPE_QUERY_WILDCARD:
-			return new WildcardQuery( readTerm() );
-		case TYPE_QUERY_TERM_RANGE:
-			{
-				String field = readUTF();
-				BytesRef lowerTerm = readBytesRef();
-				BytesRef upperTerm = readBytesRef();
-				boolean includeLower = readBoolean();
-				boolean includeUpper = readBoolean();
-				return new TermRangeQuery(field,lowerTerm,upperTerm,includeLower,includeUpper);
-			}
-		case TYPE_QUERY_PHRASE:
-			{
-				PhraseQuery query = new PhraseQuery();
-				int n = readInt();
-				for( int i=0; i<n; i++ ) {
-					Term term = readTerm();
-					int position = readInt();
-					query.add(term,position);
-				}
-				return query;
-			}
-		case TYPE_QUERY_NUMERIC_RANGE:
-			{
-				String field = readUTF();
-				Number min = (Number)readObject();
-				Number max = (Number)readObject();
-				boolean minInclusive = readBoolean();
-				boolean maxInclusive = readBoolean();
-				Number n = min!=null ? min : max;
-				if( n instanceof Integer )
-					return NumericRangeQuery.newIntRange(field,(Integer)min,(Integer)max,minInclusive,maxInclusive);
-				if( n instanceof Long )
-					return NumericRangeQuery.newLongRange(field,(Long)min,(Long)max,minInclusive,maxInclusive);
-				if( n instanceof Float )
-					return NumericRangeQuery.newFloatRange(field,(Float)min,(Float)max,minInclusive,maxInclusive);
-				if( n instanceof Double )
-					return NumericRangeQuery.newDoubleRange(field,(Double)min,(Double)max,minInclusive,maxInclusive);
-				throw new RuntimeException("bad numeric type for "+n);
-			}
-		case TYPE_QUERY_BOOLEAN:
-			{
-				BooleanQuery query = new BooleanQuery();
-				int n = readInt();
-				for( int i=0; i<n; i++ ) {
-					Query subquery = readQuery();
-					BooleanClause.Occur occur = BooleanClause.Occur.valueOf( readUTF() );
-					query.add(subquery,occur);
-				}
-				return query;
-			}
-		default:
-			throw new RuntimeException("invalid type "+type);
-		}
-	}
-
 	public void writeByteArray(byte[] bytes) throws IOException {
 		writeInt(bytes.length);
 		write(bytes);
 	}
 
-	public byte[] readByteArray() throws IOException {
-		int len = readInt();
-		byte[] bytes = new byte[len];
-		readFully(bytes);
-		return bytes;
-	}
-
 	public void writeList(List list) throws IOException {
 		writeInt(list.size());
 		for( Object obj : list ) {
@@ -299,15 +240,6 @@
 		}
 	}
 
-	public List readList() throws IOException {
-		final int size = readInt();
-		List list = new ArrayList(size);
-		for( int i=0; i<size; i++ ) {
-			list.add( readObject() );
-		}
-		return list;
-	}
-
 	public void writeMap(Map map) throws IOException {
 		writeInt(map.size());
 		for( Object obj : map.entrySet() ) {
@@ -317,43 +249,55 @@
 		}
 	}
 
-	public Map readMap() throws IOException {
-		final int size = readInt();
-		Map map = new LinkedHashMap();
-		for( int i=0; i<size; i++ ) {
-			Object key = readObject();
-			Object value = readObject();
-			map.put(key,value);
-		}
-		return map;
-	}
-
 	public void writeQuery(Query query) throws IOException {
 		writeObject(query);
 	}
 
-	public Query readQuery() throws IOException {
-		return (Query)readObject();
-	}
-
 	public void writeBytesRef(BytesRef br) throws IOException {
 		writeInt(br.length);
 		write(br.bytes,0,br.length);
 	}
 
-	public BytesRef readBytesRef() throws IOException {
-		return new BytesRef( readByteArray() );
-	}
-
 	public void writeTerm(Term term) throws IOException {
 		writeUTF(term.field());
 		writeBytesRef( term.bytes() );
 	}
 
-	public Term readTerm() throws IOException {
-		String key = readUTF();
-		BytesRef value = readBytesRef();
-		return new Term(key,value);
+
+	public void writeByte(int v) throws IOException {
+		out.writeByte(v);
+	}
+
+	public void writeInt(int v) throws IOException {
+		out.writeInt(v);
+	}
+
+	public void writeLong(long v) throws IOException {
+		out.writeLong(v);
+	}
+
+	public void writeFloat(float v) throws IOException {
+		out.writeFloat(v);
+	}
+
+	public void writeDouble(double v) throws IOException {
+		out.writeDouble(v);
+	}
+
+	public void writeBoolean(boolean v) throws IOException {
+		out.writeBoolean(v);
+	}
+
+	public void writeUTF(String s) throws IOException {
+		out.writeUTF(s);
+	}
+
+	public void write(byte[] b) throws IOException {
+		out.write(b);
+	}
+
+	public void write(byte[] b, int off, int len) throws IOException {
+		out.write(b,off,len);
 	}
 
 }
diff -r c7b86342857f -r 7d145095cc0b src/goodjava/lucene/logging/LogInputStream.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/goodjava/lucene/logging/LogInputStream.java	Sun Apr 19 20:42:26 2020 -0600
@@ -0,0 +1,159 @@
+package goodjava.lucene.logging;
+
+import java.io.InputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.LinkedHashMap;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.WildcardQuery;
+import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.NumericRangeQuery;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.util.BytesRef;
+import goodjava.logging.Logger;
+import goodjava.logging.LoggerFactory;
+
+
+public class LogInputStream extends DataInputStream {
+	private static final Logger logger = LoggerFactory.getLogger(LogInputStream.class);
+
+	public LogInputStream(InputStream in) {
+		super(in);
+	}
+
+	public Object readObject() throws IOException {
+		int type = readByte();
+		return readObject(type);
+	}
+
+	protected Object readObject(int type) throws IOException {
+		switch(type) {
+		case LogFile.TYPE_NULL:
+			return null;
+		case LogFile.TYPE_STRING:
+			return readUTF();
+		case LogFile.TYPE_INT:
+			return readInt();
+		case LogFile.TYPE_LONG:
+			return readLong();
+		case LogFile.TYPE_FLOAT:
+			return readFloat();
+		case LogFile.TYPE_DOUBLE:
+			return readDouble();
+		case LogFile.TYPE_BYTES:
+			return readByteArray();
+		case LogFile.TYPE_LIST:
+			return readList();
+		case LogFile.TYPE_QUERY_MATCH_ALL_DOCS:
+			return new MatchAllDocsQuery();
+		case LogFile.TYPE_QUERY_TERM:
+			return new TermQuery( readTerm() );
+		case LogFile.TYPE_QUERY_PREFIX:
+			return new PrefixQuery( readTerm() );
+		case LogFile.TYPE_QUERY_WILDCARD:
+			return new WildcardQuery( readTerm() );
+		case LogFile.TYPE_QUERY_TERM_RANGE:
+			{
+				String field = readUTF();
+				BytesRef lowerTerm = readBytesRef();
+				BytesRef upperTerm = readBytesRef();
+				boolean includeLower = readBoolean();
+				boolean includeUpper = readBoolean();
+				return new TermRangeQuery(field,lowerTerm,upperTerm,includeLower,includeUpper);
+			}
+		case LogFile.TYPE_QUERY_PHRASE:
+			{
+				PhraseQuery query = new PhraseQuery();
+				int n = readInt();
+				for( int i=0; i<n; i++ ) {
+					Term term = readTerm();
+					int position = readInt();
+					query.add(term,position);
+				}
+				return query;
+			}
+		case LogFile.TYPE_QUERY_NUMERIC_RANGE:
+			{
+				String field = readUTF();
+				Number min = (Number)readObject();
+				Number max = (Number)readObject();
+				boolean minInclusive = readBoolean();
+				boolean maxInclusive = readBoolean();
+				Number n = min!=null ? min : max;
+				if( n instanceof Integer )
+					return NumericRangeQuery.newIntRange(field,(Integer)min,(Integer)max,minInclusive,maxInclusive);
+				if( n instanceof Long )
+					return NumericRangeQuery.newLongRange(field,(Long)min,(Long)max,minInclusive,maxInclusive);
+				if( n instanceof Float )
+					return NumericRangeQuery.newFloatRange(field,(Float)min,(Float)max,minInclusive,maxInclusive);
+				if( n instanceof Double )
+					return NumericRangeQuery.newDoubleRange(field,(Double)min,(Double)max,minInclusive,maxInclusive);
+				throw new RuntimeException("bad numeric type for "+n);
+			}
+		case LogFile.TYPE_QUERY_BOOLEAN:
+			{
+				BooleanQuery query = new BooleanQuery();
+				int n = readInt();
+				for( int i=0; i<n; i++ ) {
+					Query subquery = readQuery();
+					BooleanClause.Occur occur = BooleanClause.Occur.valueOf( readUTF() );
+					query.add(subquery,occur);
+				}
+				return query;
+			}
+		default:
+			throw new RuntimeException("invalid type "+type);
+		}
+	}
+
+	public byte[] readByteArray() throws IOException {
+		int len = readInt();
+		byte[] bytes = new byte[len];
+		readFully(bytes);
+		return bytes;
+	}
+
+	public List readList() throws IOException {
+		final int size = readInt();
+		List list = new ArrayList(size);
+		for( int i=0; i<size; i++ ) {
+			list.add( readObject() );
+		}
+		return list;
+	}
+
+	public Map readMap() throws IOException {
+		final int size = readInt();
+		Map map = new LinkedHashMap();
+		for( int i=0; i<size; i++ ) {
+			Object key = readObject();
+			Object value = readObject();
+			map.put(key,value);
+		}
+		return map;
+	}
+
+	public Query readQuery() throws IOException {
+		return (Query)readObject();
+	}
+
+	public BytesRef readBytesRef() throws IOException {
+		return new BytesRef( readByteArray() );
+	}
+
+	public Term readTerm() throws IOException {
+		String key = readUTF();
+		BytesRef value = readBytesRef();
+		return new Term(key,value);
+	}
+
+}
diff -r c7b86342857f -r 7d145095cc0b src/goodjava/lucene/logging/LoggingIndexWriter.java
--- a/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sat Apr 18 11:02:18 2020 -0600
+++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sun Apr 19 20:42:26 2020 -0600
@@ -16,10 +16,14 @@
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.Sort;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
 import goodjava.io.IoUtils;
@@ -63,31 +67,34 @@
 						logs.add( new LogFile(file,"rwd") );
 					}
 					deleteUnusedFiles();
-					log().gotoEnd();
 					return;
 				}
 			} finally {
 				dis.close();
 			}
 		}
+		newLogs();
+	}
+
+	public synchronized boolean isMerging() {
+		return isMerging;
+	}
+
+	private synchronized void isNotMerging() {
+		isMerging = false;
+	}
+
+	public synchronized void newLogs() throws IOException {
+		if( isMerging )
+			throw new RuntimeException("merging");
+		logger.info("building new logs");
+		logs.clear();
 		for( int i=0; i<2; i++ ) {
 			logs.add( newLogFile() );
 		}
-		isMerging = true;
-		new Thread(new Runnable(){public void run(){
-			try {
-				logLucene( System.currentTimeMillis(), logs.get(0), indexWriter );
-				synchronized(LoggingIndexWriter.this) {
-					writeIndex();
-				}
-			} catch(IOException e) {
-				throw new RuntimeException(e);
-			} finally {
-				synchronized(LoggingIndexWriter.this) {
-					isMerging = false;
-				}
-			}
-		}}).start();
+		logLucene( System.currentTimeMillis(), logs.get(0), indexWriter );
+		writeIndex();
+		logger.info("done building new logs");
 	}
 
 	private static void logLucene(long time,LogFile log,LuceneIndexWriter indexWriter) throws IOException {
@@ -149,8 +156,7 @@
 		logger.info("merge");
 		LogFile first = logs.get(0);
 		LogFile second = logs.get(1);
-		second.gotoEnd();
-		long lastTime = second.readLong();
+		long lastTime = second.file.lastModified();
 		File dirFile = new File(logDir,"merge");
 		if( dirFile.exists() )
 			throw new RuntimeException();
@@ -163,53 +169,144 @@
 		logLucene( lastTime, merge, mergeWriter );
 		mergeWriter.close();
 		synchronized(this) {
-			check();
+			//check();
 			logs.remove(0);
 			logs.set(0,merge);
 			writeIndex();
-			check();
+			//check(null);
 		}
 	}
 	private final Runnable mergeLogs = new Runnable() { public void run() {
 		try {
 			mergeLogs();
-/*
 		} catch(IOException e) {
 			throw new RuntimeException(e);
-*/
-		} catch(Exception e) {
-			e.printStackTrace();
-			System.exit(-1);
 		} finally {
-			synchronized(LoggingIndexWriter.this) {
-				isMerging = false;
-			}
+			isNotMerging();
 		}
 	} };
 
-	private void check() throws IOException {
-		File dirFile = new File(logDir,"check");
-		if( dirFile.exists() )
-			throw new RuntimeException();
-		Directory dir = FSDirectory.open(dirFile);
-		LuceneIndexWriter checkWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig );
-		playLog(checkWriter);
-		int nCheck = numDocs(checkWriter);
-		int nOrig = numDocs(indexWriter);
-		if( nCheck != nOrig ) {
-			logger.error("nCheck = "+nCheck);
-			logger.error("nOrig = "+nOrig);
-			//new Exception().printStackTrace();
-			Thread.dumpStack();
-			System.out.println();
-			System.out.println("indexWriter");
-			dump(indexWriter);
-			System.out.println("checkWriter");
-			dump(checkWriter);
-			System.exit(-1);
+	private static class DocIter {
+		final IndexReader reader;
+		final TopDocs td;
+		final int n;
+		int i = 0;
+
+		DocIter(IndexReader reader,Query query,Sort sort) throws IOException {
+			this.reader = reader;
+			IndexSearcher searcher = new IndexSearcher(reader);
+			this.td = searcher.search(query,10000000,sort);
+			this.n = td.scoreDocs.length;
+			if( td.totalHits != n )
+				throw new RuntimeException();
+		}
+
+		Document next() throws IOException {
+			return i < n ? reader.document(td.scoreDocs[i++].doc) : null;
+		}
+	}
+
+	public void check(SortField sortField) throws IOException {
+		IndexReader indexReader;
+		List<LogFile> logs;
+		synchronized(this) {
+			if( isMerging ) {
+				logger.warn("is merging, check aborted");
+				return;
+			}
+			isMerging = true;
+			indexReader = indexWriter.openReader();
+			logs = new ArrayList<LogFile>(this.logs);
+			int i = logs.size() - 1;
+			LogFile last = logs.get(i);
+			logs.set(i,last.snapshot());
 		}
-		checkWriter.close();
-		IoUtils.deleteRecursively(dirFile);
+		try {
+			logger.info("check start");
+			indexWriter.check();
+			File dirFile = new File(logDir,"check");
+			IoUtils.deleteRecursively(dirFile);
+			Directory dir = FSDirectory.open(dirFile);
+			LuceneIndexWriter checkWriter = new LuceneIndexWriter( indexWriter.luceneVersion, dir, indexWriter.goodConfig );
+			playLogs(logs,checkWriter);
+			logger.info("check lucene");
+			IndexReader checkReader = checkWriter.openReader();
+			if( sortField == null ) {
+				int nCheck = checkReader.numDocs();
+				int nOrig = indexReader.numDocs();
+				if( nCheck != nOrig ) {
+					logger.error("numDocs mismatch: lucene="+nOrig+" logs="+nCheck);
+				}
+				logger.info("numDocs="+nOrig);
+				if( hash(indexReader) != hash(checkReader) ) {
+					logger.error("hash mismatch");
+				}
+			} else {
+				Sort sort = new Sort(sortField);
+				String sortFieldName = sortField.getField();
+				Query query = new PrefixQuery(new Term(sortFieldName));
+				DocIter origIter = new DocIter(indexReader,query,sort);
+				DocIter checkIter = new DocIter(checkReader,query,sort);
+				Map<String,Object> origFields = LuceneUtils.toMap(origIter.next());
+				Map<String,Object> checkFields = LuceneUtils.toMap(checkIter.next());
+				while( origFields!=null && checkFields!=null ) {
+					Comparable origFld = (Comparable)origFields.get(sortFieldName);
+					Comparable checkFld = (Comparable)checkFields.get(sortFieldName);
+					int cmp = origFld.compareTo(checkFld);
+					if( cmp==0 ) {
+						if( !origFields.equals(checkFields) ) {
+							logger.error(sortFieldName+" "+origFld+" not equal");
+							logger.error("lucene = "+origFields);
+							logger.error("logs = "+checkFields);
+						}
+						origFields = LuceneUtils.toMap(origIter.next());
+						checkFields = LuceneUtils.toMap(checkIter.next());
+					} else if( cmp < 0 ) {
+						logger.error(sortFieldName+" "+origFld+" found in lucene but not logs");
+						origFields = LuceneUtils.toMap(origIter.next());
+					} else {  // >
+						logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene");
+						checkFields = LuceneUtils.toMap(checkIter.next());
+					}
+				}
+				while( origFields!=null ) {
+					Comparable origFld = (Comparable)origFields.get(sortFieldName);
+					logger.error(sortFieldName+" "+origFld+" found in lucene but not logs");
+					origFields = LuceneUtils.toMap(origIter.next());
+				}
+				while( checkFields!=null ) {
+					Comparable checkFld = (Comparable)checkFields.get(sortFieldName);
+					logger.error(sortFieldName+" "+checkFld+" found in logs but not lucene");
+					checkFields = LuceneUtils.toMap(checkIter.next());
+				}
+				//logger.info("check done");
+			}
+			checkReader.close();
+			checkWriter.close();
+			IoUtils.deleteRecursively(dirFile);
+			logger.info("check done");
+		} finally {
+			indexReader.close();
+			isNotMerging();
+		}
+	}
+
+	private static abstract class HashCollector extends GoodCollector {
+		int total = 0;
+	}
+
+	private static int hash(IndexReader reader) throws IOException {
+		final IndexSearcher searcher = new IndexSearcher(reader);
+		Query query = new MatchAllDocsQuery();
+		HashCollector col = new HashCollector() {
+			public void collectDoc(int iDoc) throws IOException {
+				Document doc = searcher.doc(iDoc);
+				Map<String,Object> storedFields = LuceneUtils.toMap(doc);
+				total += storedFields.hashCode();
+			}
+		};
+		searcher.search(query,col);
+		return col.total;
 	}
 
 	private LogFile log() {
@@ -228,22 +325,21 @@
 		log.commit();
 		if( isMerging )
 			return;
-		if( log.length() > logs.get(0).length() ) {
-			log.writeLong( System.currentTimeMillis() );
+		if( log.end() > logs.get(0).end() ) {
 			logs.add( newLogFile() );
 			writeIndex();
 		}
 		if( logs.size() > 3 ) {
 			isMerging = true;
-//			new Thread(mergeLogs).start();
-			mergeLogs.run();
+			new Thread(mergeLogs).start();
+//			mergeLogs.run();
 		}
 	}
 
 	public synchronized void rollback() throws IOException {
 		indexWriter.rollback();
 		LogFile log = log();
-		log.gotoEnd();
+		log.rollback();
 	}
 
 	public synchronized void deleteAll() throws IOException {
@@ -283,11 +379,11 @@
 		log.writeByte(op);
 	}
 
-	public synchronized void playLog() throws IOException {
-		playLog(indexWriter);
+	public synchronized void playLogs() throws IOException {
+		playLogs(logs,indexWriter);
 	}
 
-	private void playLog(LuceneIndexWriter indexWriter) throws IOException {
+	private static void playLogs(List<LogFile> logs,LuceneIndexWriter indexWriter) throws IOException {
 		if( numDocs(indexWriter) != 0 )
 			throw new RuntimeException ("not empty");
 		for( LogFile log : logs ) {
@@ -304,32 +400,32 @@
 	}
 
 	private static void playLog(LogFile log,LuceneIndexWriter indexWriter) throws IOException {
-		log.gotoStart();
-		while( log.hasMore() ) {
-			playOp(log,indexWriter);
+		LogInputStream in = log.input();
+		while( in.available() > 0 ) {
+			playOp(in,indexWriter);
 		}
 	}
 
-	private static void playOp(LogFile log,LuceneIndexWriter indexWriter) throws IOException {
-		log.readLong();  // time
-		int op = log.readByte();
+	private static void playOp(LogInputStream in,LuceneIndexWriter indexWriter) throws IOException {
+		in.readLong();  // time
+		int op = in.readByte();
 		switch(op) {
 		case OP_DELETE_ALL:
 			indexWriter.deleteAll();
 			return;
 		case OP_DELETE_DOCUMENTS:
-			indexWriter.deleteDocuments( log.readQuery() );
+			indexWriter.deleteDocuments( in.readQuery() );
 			return;
 		case OP_ADD_DOCUMENT:
 			{
-				Map storedFields = log.readMap();
+				Map storedFields = in.readMap();
 				indexWriter.addDocument(storedFields);
 				return;
 			}
 		case OP_UPDATE_DOCUMENT:
 			{
-				String keyFieldName = log.readUTF();
-				Map storedFields = log.readMap();
+				String keyFieldName = in.readUTF();
+				Map storedFields = in.readMap();
 				indexWriter.updateDocument(keyFieldName,storedFields);
 				return;
 			}