changeset 1508:86c5e7000ecf

lucene.backup checksum
author Franklin Schmidt <fschmidt@gmail.com>
date Sat, 16 May 2020 17:56:02 -0600
parents c8f4867fd083
children 0ba144491a42
files src/goodjava/io/IoUtils.java src/goodjava/lucene/backup/Backup.java src/goodjava/lucene/backup/BackupIndexWriter.java src/goodjava/lucene/logging/LogFile.java src/goodjava/lucene/logging/LoggingIndexWriter.java src/luan/modules/IoLuan.java
diffstat 6 files changed, 68 insertions(+), 14 deletions(-) [+]
line wrap: on
line diff
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/io/IoUtils.java
--- a/src/goodjava/io/IoUtils.java	Sat May 16 12:33:41 2020 +0300
+++ b/src/goodjava/io/IoUtils.java	Sat May 16 17:56:02 2020 -0600
@@ -57,6 +57,16 @@
 		in.close();
 	}
 
+	public static long checksum(InputStream in) throws IOException {
+		long cs = 0;
+		int c;
+		while( (c=in.read()) != -1 ) {
+			cs = 31 * cs + c;
+		}
+		in.close();
+		return cs;
+	}
+
 
 	static {
 		// undo restrictions of modern scum
@@ -71,4 +81,4 @@
 		return (SSLServerSocketFactory)SSLServerSocketFactory.getDefault();
 	}
 
-}
\ No newline at end of file
+}
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/backup/Backup.java
--- a/src/goodjava/lucene/backup/Backup.java	Sat May 16 12:33:41 2020 +0300
+++ b/src/goodjava/lucene/backup/Backup.java	Sat May 16 17:56:02 2020 -0600
@@ -45,6 +45,8 @@
 		} else if( call.cmd.equals("add") || call.cmd.equals("append")  ) {
 			fileName = (String)call.args[2];
 			File f = new File(dir,fileName);
+			if( call.cmd.equals("add") )
+				IoUtils.delete(f);
 			LogFile log = new LogFile(f);
 			LogOutputStream out = log.output();
 			IoUtils.copyAll(call.in,out);
@@ -78,6 +80,15 @@
 				result = new RpcResult("incomplete",name,logEnd);
 				break;
 			}
+			Object checksumObj = fileInfo.get("checksum");
+			if( checksumObj != null ) {
+				long checksum = (Long)checksumObj;
+				if( log.checksum() != checksum ) {
+					index.delete();
+					result = new RpcResult("bad_checksum",name);
+					break;
+				}
+			}
 		}
 		if( call.cmd.equals("add") ) {
 			boolean complete = true;
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/backup/BackupIndexWriter.java
--- a/src/goodjava/lucene/backup/BackupIndexWriter.java	Sat May 16 12:33:41 2020 +0300
+++ b/src/goodjava/lucene/backup/BackupIndexWriter.java	Sat May 16 17:56:02 2020 -0600
@@ -12,6 +12,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
 import javax.net.ssl.SSLSocket;
+import org.apache.lucene.search.SortField;
 import goodjava.io.IoUtils;
 import goodjava.rpc.RpcClient;
 import goodjava.rpc.RpcCall;
@@ -56,6 +57,11 @@
 		}
 	}
 
+	protected void doCheck(SortField sortField) throws IOException {
+		super.doCheck(sortField);
+		runSyncWithChecksum();
+	}
+
 	public void runSync() {
 		try {
 			exec.submit(sync).get();
@@ -64,17 +70,35 @@
 		}
 	}
 
+	public void runSyncWithChecksum() {
+		try {
+			exec.submit(syncWithChecksum).get();
+		} catch(Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
 	private final Runnable sync = new Runnable() {
 		public void run() {
 			try {
-				sync();
+				sync(false);
 			} catch(IOException e) {
 				throw new RuntimeException(e);
 			}
 		}
 	};
 
-	private void sync() throws IOException {
+	private final Runnable syncWithChecksum = new Runnable() {
+		public void run() {
+			try {
+				sync(true);
+			} catch(IOException e) {
+				throw new RuntimeException(e);
+			}
+		}
+	};
+
+	private void sync(boolean withChecksum) throws IOException {
 		List<LogFile> logs = new ArrayList<LogFile>();
 		synchronized(this) {
 			isSyncPending = false;
@@ -91,6 +115,8 @@
 			Map fileInfo = new HashMap();
 			fileInfo.put("name",log.file.getName());
 			fileInfo.put("end",log.end());
+			if( withChecksum )
+				fileInfo.put("checksum",log.checksum());
 			logInfo.add(fileInfo);
 			logMap.put(log.file.getName(),log);
 		}
@@ -105,8 +131,10 @@
 					String status = (String)result.returnValues[0];
 					if( status.equals("ok") ) {
 						break;
-					} else if( status.equals("missing") ) {
+					} else if( status.equals("missing") || status.equals("bad_checksum") ) {
 						String fileName = (String)result.returnValues[1];
+						if( status.equals("bad_checksum") )
+							logger.error("bad_checksum "+fileName);
 						LogFile log = logMap.get(fileName);
 						long len = log.end() - 8;
 						InputStream in = log.input();
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/logging/LogFile.java
--- a/src/goodjava/lucene/logging/LogFile.java	Sat May 16 12:33:41 2020 +0300
+++ b/src/goodjava/lucene/logging/LogFile.java	Sat May 16 17:56:02 2020 -0600
@@ -12,6 +12,7 @@
 import goodjava.logging.LoggerFactory;
 import goodjava.io.FixedLengthInputStream;
 import goodjava.io.BufferedInputStream;
+import goodjava.io.IoUtils;
 
 
 public class LogFile {
@@ -65,6 +66,10 @@
 		return new LogInputStream(in);
 	}
 
+	public long checksum() throws IOException {
+		return IoUtils.checksum(input());
+	}
+
 	static final int TYPE_NULL = 0;
 	static final int TYPE_STRING = 1;
 	static final int TYPE_INT = 2;
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/logging/LoggingIndexWriter.java
--- a/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sat May 16 12:33:41 2020 +0300
+++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java	Sat May 16 17:56:02 2020 -0600
@@ -230,13 +230,21 @@
 	public void check(SortField sortField) throws IOException {
 		if( isChecking )
 			throw new RuntimeException("another check is running");
+		isChecking = true;
+		try {
+			doCheck(sortField);
+		} finally {
+			isChecking = false;
+		}
+	}
+
+	protected void doCheck(SortField sortField) throws IOException {
 		IndexReader indexReader;
 		List<LogInputStream> logReaders;
 		synchronized(this) {
 			indexReader = indexWriter.openReader();
 			logReaders = logReaders(logs);
 		}
-		isChecking = true;
 		try {
 			logger.info("check start");
 			indexWriter.check();
@@ -302,7 +310,6 @@
 			IoUtils.deleteRecursively(dirFile);
 			logger.info("check done");
 		} finally {
-			isChecking = false;
 			indexReader.close();
 		}
 	}
diff -r c8f4867fd083 -r 86c5e7000ecf src/luan/modules/IoLuan.java
--- a/src/luan/modules/IoLuan.java	Sat May 16 12:33:41 2020 +0300
+++ b/src/luan/modules/IoLuan.java	Sat May 16 17:56:02 2020 -0600
@@ -205,14 +205,7 @@
 		}
 
 		public long checksum() throws IOException, LuanException {
-			long cs = 0;
-			InputStream in = new BufferedInputStream(inputStream());
-			int c;
-			while( (c=in.read()) != -1 ) {
-				cs = 31 * cs + c;
-			}
-			in.close();
-			return cs;
+			return IoUtils.checksum( new BufferedInputStream(inputStream()) );
 		}
 
 		public String charset() {