Mercurial Hosting > luan
annotate src/goodjava/lucene/backup/BackupIndexWriter.java @ 1801:32e77b071e09
webserver logging
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Tue, 12 Mar 2024 09:28:39 -0600 |
parents | 1578324d2aac |
children |
rev | line source |
---|---|
1488 | 1 package goodjava.lucene.backup; |
2 | |
3 import java.io.File; | |
1499 | 4 import java.io.InputStream; |
1674 | 5 import java.io.OutputStream; |
6 import java.io.BufferedOutputStream; | |
7 import java.io.FileOutputStream; | |
1488 | 8 import java.io.IOException; |
1499 | 9 import java.net.Socket; |
1672 | 10 import java.net.ConnectException; |
1488 | 11 import java.util.List; |
12 import java.util.ArrayList; | |
1499 | 13 import java.util.Map; |
14 import java.util.HashMap; | |
15 import java.util.Arrays; | |
1504 | 16 import java.util.concurrent.Executors; |
17 import java.util.concurrent.ExecutorService; | |
1672 | 18 import java.util.concurrent.ExecutionException; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
19 import org.apache.lucene.search.SortField; |
1488 | 20 import goodjava.io.IoUtils; |
1499 | 21 import goodjava.rpc.RpcClient; |
22 import goodjava.rpc.RpcCall; | |
23 import goodjava.rpc.RpcResult; | |
24 import goodjava.rpc.RpcException; | |
1799 | 25 import goodjava.rpc.RpcError; |
1488 | 26 import goodjava.lucene.api.LuceneIndexWriter; |
27 import goodjava.lucene.logging.LoggingIndexWriter; | |
28 import goodjava.lucene.logging.LogFile; | |
1672 | 29 import goodjava.logging.Logger; |
30 import goodjava.logging.LoggerFactory; | |
1488 | 31 |
32 | |
1672 | 33 public final class BackupIndexWriter extends LoggingIndexWriter { |
1499 | 34 private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class); |
35 public static String[] backupDomains; | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
36 private final String domain; |
1488 | 37 private final String name; |
38 private final File dir; | |
1499 | 39 private boolean isSyncPending = false; |
1504 | 40 private final ExecutorService exec = Executors.newSingleThreadExecutor(); |
1488 | 41 |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
42 public BackupIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime,String domain,String name) |
1548 | 43 throws IOException |
44 { | |
45 super(indexWriter,logDir,logTime); | |
1499 | 46 if( backupDomains == null ) |
47 throw new RuntimeException("must set backupDomains"); | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
48 this.domain = domain; |
1488 | 49 this.name = name; |
50 File f = new File(System.getProperty("java.io.tmpdir")); | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
51 dir = new File(f,"goodjava.lucene/"+domain+"~"+name); |
1501 | 52 IoUtils.mkdirs(dir); |
1488 | 53 } |
54 | |
1672 | 55 @Override public synchronized void close() throws IOException { |
1504 | 56 super.close(); |
57 exec.shutdown(); | |
58 } | |
59 | |
1672 | 60 @Override public synchronized void commit() throws IOException { |
1488 | 61 super.commit(); |
1499 | 62 //sync(); |
63 if( !isSyncPending ) { | |
1799 | 64 exec.execute(syncR); |
1499 | 65 isSyncPending = true; |
1488 | 66 } |
67 } | |
68 | |
1672 | 69 @Override protected boolean doCheck(SortField sortField) throws IOException { |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
70 boolean ok = super.doCheck(sortField); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
71 if( ok ) |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
72 runSyncWithChecksum(); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1512
diff
changeset
|
73 return ok; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
74 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
75 |
1499 | 76 public void runSync() { |
1799 | 77 execR(syncR); |
1499 | 78 } |
79 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
80 public void runSyncWithChecksum() { |
1799 | 81 execR(syncWithChecksum); |
1672 | 82 } |
83 | |
1799 | 84 private void execR(Runnable r) { |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
85 try { |
1672 | 86 exec.submit(r).get(); |
87 } catch(InterruptedException e) { | |
88 throw new RuntimeException(e); | |
89 } catch(ExecutionException e) { | |
90 Throwable cause = e.getCause(); | |
91 if( cause instanceof RuntimeException ) | |
92 throw (RuntimeException)cause; | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
93 throw new RuntimeException(e); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
94 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
95 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
96 |
1799 | 97 private final Runnable syncR = new Runnable() { |
1504 | 98 public void run() { |
1499 | 99 try { |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
100 sync(false); |
1672 | 101 } catch(ConnectException e) { |
102 logger.error("sync failed: "+e.getMessage()); | |
1499 | 103 } catch(IOException e) { |
104 throw new RuntimeException(e); | |
1799 | 105 } catch(RpcError e) { |
106 logger.error("",e); | |
107 throw e; | |
1499 | 108 } |
109 } | |
110 }; | |
111 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
112 private final Runnable syncWithChecksum = new Runnable() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
113 public void run() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
114 try { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
115 sync(true); |
1672 | 116 } catch(ConnectException e) { |
117 logger.error("syncWithChecksum failed: "+e.getMessage()); | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
118 } catch(IOException e) { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
119 throw new RuntimeException(e); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
120 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
121 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
122 }; |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
123 |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
124 private void sync(boolean withChecksum) throws IOException { |
1499 | 125 List<LogFile> logs = new ArrayList<LogFile>(); |
126 synchronized(this) { | |
127 isSyncPending = false; | |
1500 | 128 clearDir(); |
1499 | 129 for( LogFile log : this.logs ) { |
130 File f = new File(dir,log.file.getName()); | |
131 IoUtils.link(log.file,f); | |
132 logs.add( new LogFile(f) ); | |
133 } | |
134 } | |
135 List logInfo = new ArrayList(); | |
136 Map<String,LogFile> logMap = new HashMap<String,LogFile>(); | |
137 for( LogFile log : logs ) { | |
138 Map fileInfo = new HashMap(); | |
139 fileInfo.put("name",log.file.getName()); | |
140 fileInfo.put("end",log.end()); | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
141 if( withChecksum ) |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
142 fileInfo.put("checksum",log.checksum()); |
1499 | 143 logInfo.add(fileInfo); |
144 logMap.put(log.file.getName(),log); | |
145 } | |
146 for( String backupDomain : backupDomains ) { | |
1509 | 147 RpcClient rpc = BackupServer.rpcClient(backupDomain); |
1499 | 148 try { |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
149 RpcCall call = new RpcCall("login",domain,name); |
1512 | 150 rpc.write(call); |
151 rpc.read(); | |
152 call = new RpcCall("check",logInfo); | |
1499 | 153 while(true) { |
154 rpc.write(call); | |
155 RpcResult result = rpc.read(); | |
1512 | 156 //logger.info(Arrays.asList(result.returnValues).toString()); |
1499 | 157 String status = (String)result.returnValues[0]; |
158 if( status.equals("ok") ) { | |
159 break; | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
160 } else if( status.equals("missing") || status.equals("bad_checksum") ) { |
1499 | 161 String fileName = (String)result.returnValues[1]; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
162 if( status.equals("bad_checksum") ) |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
163 logger.error("bad_checksum "+fileName); |
1499 | 164 LogFile log = logMap.get(fileName); |
165 long len = log.end() - 8; | |
166 InputStream in = log.input(); | |
1512 | 167 call = new RpcCall(in,len,"add",logInfo,fileName); |
1675 | 168 logger.info("add "+fileName); |
1499 | 169 } else if( status.equals("incomplete") ) { |
170 String fileName = (String)result.returnValues[1]; | |
171 long logEnd = (Long)result.returnValues[2]; | |
172 LogFile log = logMap.get(fileName); | |
173 long len = log.end() - logEnd; | |
174 InputStream in = log.input(); | |
175 in.skip(logEnd-8); | |
1512 | 176 call = new RpcCall(in,len,"append",logInfo,fileName); |
1675 | 177 // logger.info("append "+fileName); |
1499 | 178 } else |
179 throw new RuntimeException("status "+status); | |
180 } | |
181 } catch(RpcException e) { | |
1672 | 182 logger.error("",e); |
1499 | 183 } |
184 rpc.close(); | |
185 } | |
1500 | 186 clearDir(); |
187 } | |
188 | |
189 private void clearDir() throws IOException { | |
190 for( File f : dir.listFiles() ) { | |
191 IoUtils.delete(f); | |
192 } | |
1499 | 193 } |
194 | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
195 public static BackupIndexWriter newWithRestore(LuceneIndexWriter indexWriter,File logDir,long logTime,String domain,String name) |
1674 | 196 throws IOException |
197 { | |
198 if( !logDir.exists() ) { | |
199 RpcClient rpc = BackupServer.rpcClient(backupDomains[0]); | |
200 try { | |
201 RpcCall call; | |
202 RpcResult result; | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
203 call = new RpcCall("exists",domain,name); |
1674 | 204 rpc.write(call); |
205 result = rpc.read(); | |
206 boolean exists = (Boolean)result.returnValues[0]; | |
207 if( exists ) { | |
208 logger.error("restoring "+logDir+" from backup"); | |
209 File zip = File.createTempFile("luan_",".zip"); | |
210 IoUtils.delete(zip); | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
211 call = new RpcCall("login",domain,name); |
1674 | 212 rpc.write(call); |
213 rpc.read(); | |
214 call = new RpcCall("zip"); | |
215 rpc.write(call); | |
216 result = rpc.read(); | |
217 OutputStream out = new BufferedOutputStream(new FileOutputStream(zip)); | |
218 IoUtils.copyAll(result.in,out); | |
219 out.close(); | |
220 IoUtils.mkdirs(logDir); | |
221 String cmd = "unzip " + zip; | |
222 Process proc = Runtime.getRuntime().exec(cmd,null,logDir); | |
223 IoUtils.waitFor(proc); | |
224 IoUtils.delete(zip); | |
225 } | |
226 } catch(RpcException e) { | |
227 throw new RuntimeException(e); | |
228 } | |
229 rpc.close(); | |
230 } | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1675
diff
changeset
|
231 return new BackupIndexWriter(indexWriter,logDir,logTime,domain,name); |
1674 | 232 } |
233 | |
1488 | 234 } |