Mercurial Hosting > luan
annotate src/goodjava/lucene/backup/BackupIndexWriter.java @ 1516:70a55f49b98e
digest_message
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Mon, 08 Jun 2020 21:54:31 -0600 |
parents | 31b543826ca9 |
children | 634f6765830e |
rev | line source |
---|---|
1488 | 1 package goodjava.lucene.backup; |
2 | |
3 import java.io.File; | |
1499 | 4 import java.io.InputStream; |
1488 | 5 import java.io.IOException; |
1499 | 6 import java.net.Socket; |
1488 | 7 import java.util.List; |
8 import java.util.ArrayList; | |
1499 | 9 import java.util.Map; |
10 import java.util.HashMap; | |
11 import java.util.Arrays; | |
1504 | 12 import java.util.concurrent.Executors; |
13 import java.util.concurrent.ExecutorService; | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
14 import org.apache.lucene.search.SortField; |
1488 | 15 import goodjava.io.IoUtils; |
1499 | 16 import goodjava.rpc.RpcClient; |
17 import goodjava.rpc.RpcCall; | |
18 import goodjava.rpc.RpcResult; | |
19 import goodjava.rpc.RpcException; | |
1488 | 20 import goodjava.lucene.api.LuceneIndexWriter; |
1499 | 21 import goodjava.logging.Logger; |
22 import goodjava.logging.LoggerFactory; | |
1488 | 23 import goodjava.lucene.logging.LoggingIndexWriter; |
24 import goodjava.lucene.logging.LogFile; | |
25 | |
26 | |
27 public class BackupIndexWriter extends LoggingIndexWriter { | |
1499 | 28 private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class); |
29 public static String[] backupDomains; | |
1488 | 30 private final String name; |
1512 | 31 private final String password; |
1488 | 32 private final File dir; |
1499 | 33 private boolean isSyncPending = false; |
1504 | 34 private final ExecutorService exec = Executors.newSingleThreadExecutor(); |
1488 | 35 |
1512 | 36 public BackupIndexWriter(LuceneIndexWriter indexWriter,File logDir,String name,String password) throws IOException { |
1488 | 37 super(indexWriter,logDir); |
1499 | 38 if( backupDomains == null ) |
39 throw new RuntimeException("must set backupDomains"); | |
1488 | 40 this.name = name; |
1512 | 41 this.password = password; |
1488 | 42 File f = new File(System.getProperty("java.io.tmpdir")); |
43 dir = new File(f,"goodjava.lucene/"+name); | |
1501 | 44 IoUtils.mkdirs(dir); |
1488 | 45 } |
46 | |
1504 | 47 public synchronized void close() throws IOException { |
48 super.close(); | |
49 exec.shutdown(); | |
50 } | |
51 | |
1488 | 52 public synchronized void commit() throws IOException { |
53 super.commit(); | |
1499 | 54 //sync(); |
55 if( !isSyncPending ) { | |
1504 | 56 exec.execute(sync); |
1499 | 57 isSyncPending = true; |
1488 | 58 } |
59 } | |
60 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
61 protected void doCheck(SortField sortField) throws IOException { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
62 super.doCheck(sortField); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
63 runSyncWithChecksum(); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
64 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
65 |
1499 | 66 public void runSync() { |
1504 | 67 try { |
68 exec.submit(sync).get(); | |
69 } catch(Exception e) { | |
70 throw new RuntimeException(e); | |
71 } | |
1499 | 72 } |
73 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
74 public void runSyncWithChecksum() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
75 try { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
76 exec.submit(syncWithChecksum).get(); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
77 } catch(Exception e) { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
78 throw new RuntimeException(e); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
79 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
80 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
81 |
1499 | 82 private final Runnable sync = new Runnable() { |
1504 | 83 public void run() { |
1499 | 84 try { |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
85 sync(false); |
1499 | 86 } catch(IOException e) { |
87 throw new RuntimeException(e); | |
88 } | |
89 } | |
90 }; | |
91 | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
92 private final Runnable syncWithChecksum = new Runnable() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
93 public void run() { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
94 try { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
95 sync(true); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
96 } catch(IOException e) { |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
97 throw new RuntimeException(e); |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
98 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
99 } |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
100 }; |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
101 |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
102 private void sync(boolean withChecksum) throws IOException { |
1499 | 103 List<LogFile> logs = new ArrayList<LogFile>(); |
104 synchronized(this) { | |
105 isSyncPending = false; | |
1500 | 106 clearDir(); |
1499 | 107 for( LogFile log : this.logs ) { |
108 File f = new File(dir,log.file.getName()); | |
109 IoUtils.link(log.file,f); | |
110 logs.add( new LogFile(f) ); | |
111 } | |
112 } | |
113 List logInfo = new ArrayList(); | |
114 Map<String,LogFile> logMap = new HashMap<String,LogFile>(); | |
115 for( LogFile log : logs ) { | |
116 Map fileInfo = new HashMap(); | |
117 fileInfo.put("name",log.file.getName()); | |
118 fileInfo.put("end",log.end()); | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
119 if( withChecksum ) |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
120 fileInfo.put("checksum",log.checksum()); |
1499 | 121 logInfo.add(fileInfo); |
122 logMap.put(log.file.getName(),log); | |
123 } | |
124 for( String backupDomain : backupDomains ) { | |
1509 | 125 RpcClient rpc = BackupServer.rpcClient(backupDomain); |
1499 | 126 try { |
1512 | 127 RpcCall call = new RpcCall("login",name,password); |
128 rpc.write(call); | |
129 rpc.read(); | |
130 call = new RpcCall("check",logInfo); | |
1499 | 131 while(true) { |
132 rpc.write(call); | |
133 RpcResult result = rpc.read(); | |
1512 | 134 //logger.info(Arrays.asList(result.returnValues).toString()); |
1499 | 135 String status = (String)result.returnValues[0]; |
136 if( status.equals("ok") ) { | |
137 break; | |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
138 } else if( status.equals("missing") || status.equals("bad_checksum") ) { |
1499 | 139 String fileName = (String)result.returnValues[1]; |
1508
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
140 if( status.equals("bad_checksum") ) |
86c5e7000ecf
lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents:
1506
diff
changeset
|
141 logger.error("bad_checksum "+fileName); |
1499 | 142 LogFile log = logMap.get(fileName); |
143 long len = log.end() - 8; | |
144 InputStream in = log.input(); | |
1512 | 145 call = new RpcCall(in,len,"add",logInfo,fileName); |
1499 | 146 } else if( status.equals("incomplete") ) { |
147 String fileName = (String)result.returnValues[1]; | |
148 long logEnd = (Long)result.returnValues[2]; | |
149 LogFile log = logMap.get(fileName); | |
150 long len = log.end() - logEnd; | |
151 InputStream in = log.input(); | |
152 in.skip(logEnd-8); | |
1512 | 153 call = new RpcCall(in,len,"append",logInfo,fileName); |
1499 | 154 } else |
155 throw new RuntimeException("status "+status); | |
156 } | |
157 } catch(RpcException e) { | |
158 logger.warn("",e); | |
159 } | |
160 rpc.close(); | |
161 } | |
1500 | 162 clearDir(); |
163 } | |
164 | |
165 private void clearDir() throws IOException { | |
166 for( File f : dir.listFiles() ) { | |
167 IoUtils.delete(f); | |
168 } | |
1499 | 169 } |
170 | |
1488 | 171 } |