Mercurial Hosting > luan
annotate src/goodjava/lucene/backup/BackupIndexWriter.java @ 1673:1b9f9fdb3b41
remove backup password
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Sat, 28 May 2022 21:00:30 -0600 |
| parents | 8dd8c556c449 |
| children | af18eacf187c |
| rev | line source |
|---|---|
| 1488 | 1 package goodjava.lucene.backup; |
| 2 | |
| 3 import java.io.File; | |
| 1499 | 4 import java.io.InputStream; |
| 1488 | 5 import java.io.IOException; |
| 1499 | 6 import java.net.Socket; |
| 1672 | 7 import java.net.ConnectException; |
| 1488 | 8 import java.util.List; |
| 9 import java.util.ArrayList; | |
| 1499 | 10 import java.util.Map; |
| 11 import java.util.HashMap; | |
| 12 import java.util.Arrays; | |
| 1504 | 13 import java.util.concurrent.Executors; |
| 14 import java.util.concurrent.ExecutorService; | |
| 1672 | 15 import java.util.concurrent.ExecutionException; |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
16 import org.apache.lucene.search.SortField; |
| 1488 | 17 import goodjava.io.IoUtils; |
| 1499 | 18 import goodjava.rpc.RpcClient; |
| 19 import goodjava.rpc.RpcCall; | |
| 20 import goodjava.rpc.RpcResult; | |
| 21 import goodjava.rpc.RpcException; | |
| 1488 | 22 import goodjava.lucene.api.LuceneIndexWriter; |
| 23 import goodjava.lucene.logging.LoggingIndexWriter; | |
| 24 import goodjava.lucene.logging.LogFile; | |
| 1672 | 25 import goodjava.logging.Logger; |
| 26 import goodjava.logging.LoggerFactory; | |
| 1488 | 27 |
| 28 | |
| 1672 | 29 public final class BackupIndexWriter extends LoggingIndexWriter { |
| 1499 | 30 private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class); |
| 31 public static String[] backupDomains; | |
| 1488 | 32 private final String name; |
| 33 private final File dir; | |
| 1499 | 34 private boolean isSyncPending = false; |
| 1504 | 35 private final ExecutorService exec = Executors.newSingleThreadExecutor(); |
| 1488 | 36 |
|
1673
1b9f9fdb3b41
remove backup password
Franklin Schmidt <fschmidt@gmail.com>
parents:
1672
diff
changeset
|
37 public BackupIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime,String name) |
| 1548 | 38 throws IOException |
| 39 { | |
| 40 super(indexWriter,logDir,logTime); | |
| 1499 | 41 if( backupDomains == null ) |
| 42 throw new RuntimeException("must set backupDomains"); | |
| 1488 | 43 this.name = name; |
| 44 File f = new File(System.getProperty("java.io.tmpdir")); | |
| 45 dir = new File(f,"goodjava.lucene/"+name); | |
| 1501 | 46 IoUtils.mkdirs(dir); |
| 1488 | 47 } |
| 48 | |
| 1672 | 49 @Override public synchronized void close() throws IOException { |
| 1504 | 50 super.close(); |
| 51 exec.shutdown(); | |
| 52 } | |
| 53 | |
| 1672 | 54 @Override public synchronized void commit() throws IOException { |
| 1488 | 55 super.commit(); |
| 1499 | 56 //sync(); |
| 57 if( !isSyncPending ) { | |
| 1504 | 58 exec.execute(sync); |
| 1499 | 59 isSyncPending = true; |
| 1488 | 60 } |
| 61 } | |
| 62 | |
| 1672 | 63 @Override protected boolean doCheck(SortField sortField) throws IOException { |
|
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
64 boolean ok = super.doCheck(sortField); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
65 if( ok ) |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
66 runSyncWithChecksum(); |
|
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
67 return ok; |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
68 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
69 |
| 1499 | 70 public void runSync() { |
| 1672 | 71 exec(sync); |
| 1499 | 72 } |
| 73 | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
74 public void runSyncWithChecksum() { |
| 1672 | 75 exec(syncWithChecksum); |
| 76 } | |
| 77 | |
| 78 private void exec(Runnable r) { | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
79 try { |
| 1672 | 80 exec.submit(r).get(); |
| 81 } catch(InterruptedException e) { | |
| 82 throw new RuntimeException(e); | |
| 83 } catch(ExecutionException e) { | |
| 84 Throwable cause = e.getCause(); | |
| 85 if( cause instanceof RuntimeException ) | |
| 86 throw (RuntimeException)cause; | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
87 throw new RuntimeException(e); |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
88 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
89 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
90 |
| 1499 | 91 private final Runnable sync = new Runnable() { |
| 1504 | 92 public void run() { |
| 1499 | 93 try { |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
94 sync(false); |
| 1672 | 95 } catch(ConnectException e) { |
| 96 logger.error("sync failed: "+e.getMessage()); | |
| 1499 | 97 } catch(IOException e) { |
| 98 throw new RuntimeException(e); | |
| 99 } | |
| 100 } | |
| 101 }; | |
| 102 | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
103 private final Runnable syncWithChecksum = new Runnable() { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
104 public void run() { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
105 try { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
106 sync(true); |
| 1672 | 107 } catch(ConnectException e) { |
| 108 logger.error("syncWithChecksum failed: "+e.getMessage()); | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
109 } catch(IOException e) { |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
110 throw new RuntimeException(e); |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
111 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
112 } |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
113 }; |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
114 |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
115 private void sync(boolean withChecksum) throws IOException { |
| 1499 | 116 List<LogFile> logs = new ArrayList<LogFile>(); |
| 117 synchronized(this) { | |
| 118 isSyncPending = false; | |
| 1500 | 119 clearDir(); |
| 1499 | 120 for( LogFile log : this.logs ) { |
| 121 File f = new File(dir,log.file.getName()); | |
| 122 IoUtils.link(log.file,f); | |
| 123 logs.add( new LogFile(f) ); | |
| 124 } | |
| 125 } | |
| 126 List logInfo = new ArrayList(); | |
| 127 Map<String,LogFile> logMap = new HashMap<String,LogFile>(); | |
| 128 for( LogFile log : logs ) { | |
| 129 Map fileInfo = new HashMap(); | |
| 130 fileInfo.put("name",log.file.getName()); | |
| 131 fileInfo.put("end",log.end()); | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
132 if( withChecksum ) |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
133 fileInfo.put("checksum",log.checksum()); |
| 1499 | 134 logInfo.add(fileInfo); |
| 135 logMap.put(log.file.getName(),log); | |
| 136 } | |
| 137 for( String backupDomain : backupDomains ) { | |
| 1509 | 138 RpcClient rpc = BackupServer.rpcClient(backupDomain); |
| 1499 | 139 try { |
|
1673
1b9f9fdb3b41
remove backup password
Franklin Schmidt <fschmidt@gmail.com>
parents:
1672
diff
changeset
|
140 RpcCall call = new RpcCall("login",name); |
| 1512 | 141 rpc.write(call); |
| 142 rpc.read(); | |
| 143 call = new RpcCall("check",logInfo); | |
| 1499 | 144 while(true) { |
| 145 rpc.write(call); | |
| 146 RpcResult result = rpc.read(); | |
| 1512 | 147 //logger.info(Arrays.asList(result.returnValues).toString()); |
| 1499 | 148 String status = (String)result.returnValues[0]; |
| 149 if( status.equals("ok") ) { | |
| 150 break; | |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
151 } else if( status.equals("missing") || status.equals("bad_checksum") ) { |
| 1499 | 152 String fileName = (String)result.returnValues[1]; |
|
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
153 if( status.equals("bad_checksum") ) |
|
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
154 logger.error("bad_checksum "+fileName); |
| 1499 | 155 LogFile log = logMap.get(fileName); |
| 156 long len = log.end() - 8; | |
| 157 InputStream in = log.input(); | |
| 1512 | 158 call = new RpcCall(in,len,"add",logInfo,fileName); |
| 1499 | 159 } else if( status.equals("incomplete") ) { |
| 160 String fileName = (String)result.returnValues[1]; | |
| 161 long logEnd = (Long)result.returnValues[2]; | |
| 162 LogFile log = logMap.get(fileName); | |
| 163 long len = log.end() - logEnd; | |
| 164 InputStream in = log.input(); | |
| 165 in.skip(logEnd-8); | |
| 1512 | 166 call = new RpcCall(in,len,"append",logInfo,fileName); |
| 1499 | 167 } else |
| 168 throw new RuntimeException("status "+status); | |
| 169 } | |
| 170 } catch(RpcException e) { | |
| 1672 | 171 logger.error("",e); |
| 1499 | 172 } |
| 173 rpc.close(); | |
| 174 } | |
| 1500 | 175 clearDir(); |
| 176 } | |
| 177 | |
| 178 private void clearDir() throws IOException { | |
| 179 for( File f : dir.listFiles() ) { | |
| 180 IoUtils.delete(f); | |
| 181 } | |
| 1499 | 182 } |
| 183 | |
| 1488 | 184 } |