Mercurial Hosting > luan
changeset 1508:86c5e7000ecf
lucene.backup checksum
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sat, 16 May 2020 17:56:02 -0600 |
parents | c8f4867fd083 |
children | 0ba144491a42 |
files | src/goodjava/io/IoUtils.java src/goodjava/lucene/backup/Backup.java src/goodjava/lucene/backup/BackupIndexWriter.java src/goodjava/lucene/logging/LogFile.java src/goodjava/lucene/logging/LoggingIndexWriter.java src/luan/modules/IoLuan.java |
diffstat | 6 files changed, 68 insertions(+), 14 deletions(-) [+] |
line wrap: on
line diff
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/io/IoUtils.java --- a/src/goodjava/io/IoUtils.java Sat May 16 12:33:41 2020 +0300 +++ b/src/goodjava/io/IoUtils.java Sat May 16 17:56:02 2020 -0600 @@ -57,6 +57,16 @@ in.close(); } + public static long checksum(InputStream in) throws IOException { + long cs = 0; + int c; + while( (c=in.read()) != -1 ) { + cs = 31 * cs + c; + } + in.close(); + return cs; + } + static { // undo restrictions of modern scum @@ -71,4 +81,4 @@ return (SSLServerSocketFactory)SSLServerSocketFactory.getDefault(); } -} \ No newline at end of file +}
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/backup/Backup.java --- a/src/goodjava/lucene/backup/Backup.java Sat May 16 12:33:41 2020 +0300 +++ b/src/goodjava/lucene/backup/Backup.java Sat May 16 17:56:02 2020 -0600 @@ -45,6 +45,8 @@ } else if( call.cmd.equals("add") || call.cmd.equals("append") ) { fileName = (String)call.args[2]; File f = new File(dir,fileName); + if( call.cmd.equals("add") ) + IoUtils.delete(f); LogFile log = new LogFile(f); LogOutputStream out = log.output(); IoUtils.copyAll(call.in,out); @@ -78,6 +80,15 @@ result = new RpcResult("incomplete",name,logEnd); break; } + Object checksumObj = fileInfo.get("checksum"); + if( checksumObj != null ) { + long checksum = (Long)checksumObj; + if( log.checksum() != checksum ) { + index.delete(); + result = new RpcResult("bad_checksum",name); + break; + } + } } if( call.cmd.equals("add") ) { boolean complete = true;
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/backup/BackupIndexWriter.java --- a/src/goodjava/lucene/backup/BackupIndexWriter.java Sat May 16 12:33:41 2020 +0300 +++ b/src/goodjava/lucene/backup/BackupIndexWriter.java Sat May 16 17:56:02 2020 -0600 @@ -12,6 +12,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ExecutorService; import javax.net.ssl.SSLSocket; +import org.apache.lucene.search.SortField; import goodjava.io.IoUtils; import goodjava.rpc.RpcClient; import goodjava.rpc.RpcCall; @@ -56,6 +57,11 @@ } } + protected void doCheck(SortField sortField) throws IOException { + super.doCheck(sortField); + runSyncWithChecksum(); + } + public void runSync() { try { exec.submit(sync).get(); @@ -64,17 +70,35 @@ } } + public void runSyncWithChecksum() { + try { + exec.submit(syncWithChecksum).get(); + } catch(Exception e) { + throw new RuntimeException(e); + } + } + private final Runnable sync = new Runnable() { public void run() { try { - sync(); + sync(false); } catch(IOException e) { throw new RuntimeException(e); } } }; - private void sync() throws IOException { + private final Runnable syncWithChecksum = new Runnable() { + public void run() { + try { + sync(true); + } catch(IOException e) { + throw new RuntimeException(e); + } + } + }; + + private void sync(boolean withChecksum) throws IOException { List<LogFile> logs = new ArrayList<LogFile>(); synchronized(this) { isSyncPending = false; @@ -91,6 +115,8 @@ Map fileInfo = new HashMap(); fileInfo.put("name",log.file.getName()); fileInfo.put("end",log.end()); + if( withChecksum ) + fileInfo.put("checksum",log.checksum()); logInfo.add(fileInfo); logMap.put(log.file.getName(),log); } @@ -105,8 +131,10 @@ String status = (String)result.returnValues[0]; if( status.equals("ok") ) { break; - } else if( status.equals("missing") ) { + } else if( status.equals("missing") || status.equals("bad_checksum") ) { String fileName = (String)result.returnValues[1]; + if( status.equals("bad_checksum") ) + logger.error("bad_checksum "+fileName); LogFile log = logMap.get(fileName); long len = log.end() - 8; InputStream in = log.input();
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/logging/LogFile.java --- a/src/goodjava/lucene/logging/LogFile.java Sat May 16 12:33:41 2020 +0300 +++ b/src/goodjava/lucene/logging/LogFile.java Sat May 16 17:56:02 2020 -0600 @@ -12,6 +12,7 @@ import goodjava.logging.LoggerFactory; import goodjava.io.FixedLengthInputStream; import goodjava.io.BufferedInputStream; +import goodjava.io.IoUtils; public class LogFile { @@ -65,6 +66,10 @@ return new LogInputStream(in); } + public long checksum() throws IOException { + return IoUtils.checksum(input()); + } + static final int TYPE_NULL = 0; static final int TYPE_STRING = 1; static final int TYPE_INT = 2;
diff -r c8f4867fd083 -r 86c5e7000ecf src/goodjava/lucene/logging/LoggingIndexWriter.java --- a/src/goodjava/lucene/logging/LoggingIndexWriter.java Sat May 16 12:33:41 2020 +0300 +++ b/src/goodjava/lucene/logging/LoggingIndexWriter.java Sat May 16 17:56:02 2020 -0600 @@ -230,13 +230,21 @@ public void check(SortField sortField) throws IOException { if( isChecking ) throw new RuntimeException("another check is running"); + isChecking = true; + try { + doCheck(sortField); + } finally { + isChecking = false; + } + } + + protected void doCheck(SortField sortField) throws IOException { IndexReader indexReader; List<LogInputStream> logReaders; synchronized(this) { indexReader = indexWriter.openReader(); logReaders = logReaders(logs); } - isChecking = true; try { logger.info("check start"); indexWriter.check(); @@ -302,7 +310,6 @@ IoUtils.deleteRecursively(dirFile); logger.info("check done"); } finally { - isChecking = false; indexReader.close(); } }
diff -r c8f4867fd083 -r 86c5e7000ecf src/luan/modules/IoLuan.java --- a/src/luan/modules/IoLuan.java Sat May 16 12:33:41 2020 +0300 +++ b/src/luan/modules/IoLuan.java Sat May 16 17:56:02 2020 -0600 @@ -205,14 +205,7 @@ } public long checksum() throws IOException, LuanException { - long cs = 0; - InputStream in = new BufferedInputStream(inputStream()); - int c; - while( (c=in.read()) != -1 ) { - cs = 31 * cs + c; - } - in.close(); - return cs; + return IoUtils.checksum( new BufferedInputStream(inputStream()) ); } public String charset() {