Mercurial Hosting > luan
annotate src/goodjava/lucene/backup/BackupIndexWriter.java @ 1682:97cc73664ca8
improve RollingFileAppender
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 05 Jun 2022 10:51:43 -0600 |
parents | 445048f9eca7 |
children | 973d3039c421 |
rev | line source |
---|---|
1488 | 1 package goodjava.lucene.backup; |
2 | |
3 import java.io.File; | |
1499 | 4 import java.io.InputStream; |
1674 | 5 import java.io.OutputStream; |
6 import java.io.BufferedOutputStream; | |
7 import java.io.FileOutputStream; | |
1488 | 8 import java.io.IOException; |
1499 | 9 import java.net.Socket; |
1672 | 10 import java.net.ConnectException; |
1488 | 11 import java.util.List; |
12 import java.util.ArrayList; | |
1499 | 13 import java.util.Map; |
14 import java.util.HashMap; | |
15 import java.util.Arrays; | |
1504 | 16 import java.util.concurrent.Executors; |
17 import java.util.concurrent.ExecutorService; | |
1672 | 18 import java.util.concurrent.ExecutionException; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
19 import org.apache.lucene.search.SortField; |
1488 | 20 import goodjava.io.IoUtils; |
1499 | 21 import goodjava.rpc.RpcClient; |
22 import goodjava.rpc.RpcCall; | |
23 import goodjava.rpc.RpcResult; | |
24 import goodjava.rpc.RpcException; | |
1488 | 25 import goodjava.lucene.api.LuceneIndexWriter; |
26 import goodjava.lucene.logging.LoggingIndexWriter; | |
27 import goodjava.lucene.logging.LogFile; | |
1672 | 28 import goodjava.logging.Logger; |
29 import goodjava.logging.LoggerFactory; | |
1488 | 30 |
31 | |
1672 | 32 public final class BackupIndexWriter extends LoggingIndexWriter { |
1499 | 33 private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class); |
34 public static String[] backupDomains; | |
1488 | 35 private final String name; |
36 private final File dir; | |
1499 | 37 private boolean isSyncPending = false; |
1504 | 38 private final ExecutorService exec = Executors.newSingleThreadExecutor(); |
1488 | 39 |
1673
1b9f9fdb3b41
remove backup password
Franklin Schmidt <fschmidt@gmail.com>
parents:
1672
diff
changeset
|
40 public BackupIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime,String name) |
1548 | 41 throws IOException |
42 { | |
43 super(indexWriter,logDir,logTime); | |
1499 | 44 if( backupDomains == null ) |
45 throw new RuntimeException("must set backupDomains"); | |
1488 | 46 this.name = name; |
47 File f = new File(System.getProperty("java.io.tmpdir")); | |
48 dir = new File(f,"goodjava.lucene/"+name); | |
1501 | 49 IoUtils.mkdirs(dir); |
1488 | 50 } |
51 | |
1672 | 52 @Override public synchronized void close() throws IOException { |
1504 | 53 super.close(); |
54 exec.shutdown(); | |
55 } | |
56 | |
1672 | 57 @Override public synchronized void commit() throws IOException { |
1488 | 58 super.commit(); |
1499 | 59 //sync(); |
60 if( !isSyncPending ) { | |
1504 | 61 exec.execute(sync); |
1499 | 62 isSyncPending = true; |
1488 | 63 } |
64 } | |
65 | |
1672 | 66 @Override protected boolean doCheck(SortField sortField) throws IOException { |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
67 boolean ok = super.doCheck(sortField); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
68 if( ok ) |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
69 runSyncWithChecksum(); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
70 return ok; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
71 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
72 |
1499 | 73 public void runSync() { |
1672 | 74 exec(sync); |
1499 | 75 } |
76 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
77 public void runSyncWithChecksum() { |
1672 | 78 exec(syncWithChecksum); |
79 } | |
80 | |
81 private void exec(Runnable r) { | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
82 try { |
1672 | 83 exec.submit(r).get(); |
84 } catch(InterruptedException e) { | |
85 throw new RuntimeException(e); | |
86 } catch(ExecutionException e) { | |
87 Throwable cause = e.getCause(); | |
88 if( cause instanceof RuntimeException ) | |
89 throw (RuntimeException)cause; | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
90 throw new RuntimeException(e); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
91 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
92 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
93 |
1499 | 94 private final Runnable sync = new Runnable() { |
1504 | 95 public void run() { |
1499 | 96 try { |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
97 sync(false); |
1672 | 98 } catch(ConnectException e) { |
99 logger.error("sync failed: "+e.getMessage()); | |
1499 | 100 } catch(IOException e) { |
101 throw new RuntimeException(e); | |
102 } | |
103 } | |
104 }; | |
105 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
106 private final Runnable syncWithChecksum = new Runnable() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
107 public void run() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
108 try { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
109 sync(true); |
1672 | 110 } catch(ConnectException e) { |
111 logger.error("syncWithChecksum failed: "+e.getMessage()); | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
112 } catch(IOException e) { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
113 throw new RuntimeException(e); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
114 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
115 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
116 }; |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
117 |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
118 private void sync(boolean withChecksum) throws IOException { |
1499 | 119 List<LogFile> logs = new ArrayList<LogFile>(); |
120 synchronized(this) { | |
121 isSyncPending = false; | |
1500 | 122 clearDir(); |
1499 | 123 for( LogFile log : this.logs ) { |
124 File f = new File(dir,log.file.getName()); | |
125 IoUtils.link(log.file,f); | |
126 logs.add( new LogFile(f) ); | |
127 } | |
128 } | |
129 List logInfo = new ArrayList(); | |
130 Map<String,LogFile> logMap = new HashMap<String,LogFile>(); | |
131 for( LogFile log : logs ) { | |
132 Map fileInfo = new HashMap(); | |
133 fileInfo.put("name",log.file.getName()); | |
134 fileInfo.put("end",log.end()); | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
135 if( withChecksum ) |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
136 fileInfo.put("checksum",log.checksum()); |
1499 | 137 logInfo.add(fileInfo); |
138 logMap.put(log.file.getName(),log); | |
139 } | |
140 for( String backupDomain : backupDomains ) { | |
1509 | 141 RpcClient rpc = BackupServer.rpcClient(backupDomain); |
1499 | 142 try { |
1673
1b9f9fdb3b41
remove backup password
Franklin Schmidt <fschmidt@gmail.com>
parents:
1672
diff
changeset
|
143 RpcCall call = new RpcCall("login",name); |
1512 | 144 rpc.write(call); |
145 rpc.read(); | |
146 call = new RpcCall("check",logInfo); | |
1499 | 147 while(true) { |
148 rpc.write(call); | |
149 RpcResult result = rpc.read(); | |
1512 | 150 //logger.info(Arrays.asList(result.returnValues).toString()); |
1499 | 151 String status = (String)result.returnValues[0]; |
152 if( status.equals("ok") ) { | |
153 break; | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
154 } else if( status.equals("missing") || status.equals("bad_checksum") ) { |
1499 | 155 String fileName = (String)result.returnValues[1]; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
156 if( status.equals("bad_checksum") ) |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
157 logger.error("bad_checksum "+fileName); |
1499 | 158 LogFile log = logMap.get(fileName); |
159 long len = log.end() - 8; | |
160 InputStream in = log.input(); | |
1512 | 161 call = new RpcCall(in,len,"add",logInfo,fileName); |
1675 | 162 logger.info("add "+fileName); |
1499 | 163 } else if( status.equals("incomplete") ) { |
164 String fileName = (String)result.returnValues[1]; | |
165 long logEnd = (Long)result.returnValues[2]; | |
166 LogFile log = logMap.get(fileName); | |
167 long len = log.end() - logEnd; | |
168 InputStream in = log.input(); | |
169 in.skip(logEnd-8); | |
1512 | 170 call = new RpcCall(in,len,"append",logInfo,fileName); |
1675 | 171 // logger.info("append "+fileName); |
1499 | 172 } else |
173 throw new RuntimeException("status "+status); | |
174 } | |
175 } catch(RpcException e) { | |
1672 | 176 logger.error("",e); |
1499 | 177 } |
178 rpc.close(); | |
179 } | |
1500 | 180 clearDir(); |
181 } | |
182 | |
183 private void clearDir() throws IOException { | |
184 for( File f : dir.listFiles() ) { | |
185 IoUtils.delete(f); | |
186 } | |
1499 | 187 } |
188 | |
1674 | 189 public static BackupIndexWriter newWithRestore(LuceneIndexWriter indexWriter,File logDir,long logTime,String name) |
190 throws IOException | |
191 { | |
192 if( !logDir.exists() ) { | |
193 RpcClient rpc = BackupServer.rpcClient(backupDomains[0]); | |
194 try { | |
195 RpcCall call; | |
196 RpcResult result; | |
197 call = new RpcCall("exists",name); | |
198 rpc.write(call); | |
199 result = rpc.read(); | |
200 boolean exists = (Boolean)result.returnValues[0]; | |
201 if( exists ) { | |
202 logger.error("restoring "+logDir+" from backup"); | |
203 File zip = File.createTempFile("luan_",".zip"); | |
204 IoUtils.delete(zip); | |
205 call = new RpcCall("login",name); | |
206 rpc.write(call); | |
207 rpc.read(); | |
208 call = new RpcCall("zip"); | |
209 rpc.write(call); | |
210 result = rpc.read(); | |
211 OutputStream out = new BufferedOutputStream(new FileOutputStream(zip)); | |
212 IoUtils.copyAll(result.in,out); | |
213 out.close(); | |
214 IoUtils.mkdirs(logDir); | |
215 String cmd = "unzip " + zip; | |
216 Process proc = Runtime.getRuntime().exec(cmd,null,logDir); | |
217 IoUtils.waitFor(proc); | |
218 IoUtils.delete(zip); | |
219 } | |
220 } catch(RpcException e) { | |
221 throw new RuntimeException(e); | |
222 } | |
223 rpc.close(); | |
224 } | |
225 return new BackupIndexWriter(indexWriter,logDir,logTime,name); | |
226 } | |
227 | |
1488 | 228 } |