annotate src/goodjava/lucene/backup/BackupIndexWriter.java @ 1811:55d89a183c82

remove line_diff from luan
author Franklin Schmidt <fschmidt@gmail.com>
date Wed, 15 May 2024 18:02:28 -0600
parents 1578324d2aac
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
1 package goodjava.lucene.backup;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
2
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
3 import java.io.File;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
4 import java.io.InputStream;
1674
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
5 import java.io.OutputStream;
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
6 import java.io.BufferedOutputStream;
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
7 import java.io.FileOutputStream;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
8 import java.io.IOException;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
9 import java.net.Socket;
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
10 import java.net.ConnectException;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
11 import java.util.List;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
12 import java.util.ArrayList;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
13 import java.util.Map;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
14 import java.util.HashMap;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
15 import java.util.Arrays;
1504
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
16 import java.util.concurrent.Executors;
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
17 import java.util.concurrent.ExecutorService;
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
18 import java.util.concurrent.ExecutionException;
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
19 import org.apache.lucene.search.SortField;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
20 import goodjava.io.IoUtils;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
21 import goodjava.rpc.RpcClient;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
22 import goodjava.rpc.RpcCall;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
23 import goodjava.rpc.RpcResult;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
24 import goodjava.rpc.RpcException;
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
25 import goodjava.rpc.RpcError;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
26 import goodjava.lucene.api.LuceneIndexWriter;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
27 import goodjava.lucene.logging.LoggingIndexWriter;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
28 import goodjava.lucene.logging.LogFile;
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
29 import goodjava.logging.Logger;
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
30 import goodjava.logging.LoggerFactory;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
31
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
32
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
33 public final class BackupIndexWriter extends LoggingIndexWriter {
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
34 private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
35 public static String[] backupDomains;
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
36 private final String domain;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
37 private final String name;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
38 private final File dir;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
39 private boolean isSyncPending = false;
1504
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
40 private final ExecutorService exec = Executors.newSingleThreadExecutor();
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
41
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
42 public BackupIndexWriter(LuceneIndexWriter indexWriter,File logDir,long logTime,String domain,String name)
1548
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
43 throws IOException
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
44 {
736ec76bbf42 lucene log work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1538
diff changeset
45 super(indexWriter,logDir,logTime);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
46 if( backupDomains == null )
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
47 throw new RuntimeException("must set backupDomains");
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
48 this.domain = domain;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
49 this.name = name;
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
50 File f = new File(System.getProperty("java.io.tmpdir"));
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
51 dir = new File(f,"goodjava.lucene/"+domain+"~"+name);
1501
Franklin Schmidt <fschmidt@gmail.com>
parents: 1500
diff changeset
52 IoUtils.mkdirs(dir);
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
53 }
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
54
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
55 @Override public synchronized void close() throws IOException {
1504
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
56 super.close();
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
57 exec.shutdown();
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
58 }
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
59
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
60 @Override public synchronized void commit() throws IOException {
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
61 super.commit();
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
62 //sync();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
63 if( !isSyncPending ) {
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
64 exec.execute(syncR);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
65 isSyncPending = true;
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
66 }
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
67 }
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
68
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
69 @Override protected boolean doCheck(SortField sortField) throws IOException {
1538
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
70 boolean ok = super.doCheck(sortField);
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
71 if( ok )
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
72 runSyncWithChecksum();
634f6765830e use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1512
diff changeset
73 return ok;
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
74 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
75
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
76 public void runSync() {
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
77 execR(syncR);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
78 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
79
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
80 public void runSyncWithChecksum() {
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
81 execR(syncWithChecksum);
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
82 }
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
83
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
84 private void execR(Runnable r) {
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
85 try {
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
86 exec.submit(r).get();
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
87 } catch(InterruptedException e) {
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
88 throw new RuntimeException(e);
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
89 } catch(ExecutionException e) {
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
90 Throwable cause = e.getCause();
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
91 if( cause instanceof RuntimeException )
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
92 throw (RuntimeException)cause;
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
93 throw new RuntimeException(e);
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
94 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
95 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
96
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
97 private final Runnable syncR = new Runnable() {
1504
f443542d8650 threading
Franklin Schmidt <fschmidt@gmail.com>
parents: 1502
diff changeset
98 public void run() {
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
99 try {
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
100 sync(false);
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
101 } catch(ConnectException e) {
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
102 logger.error("sync failed: "+e.getMessage());
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
103 } catch(IOException e) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
104 throw new RuntimeException(e);
1799
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
105 } catch(RpcError e) {
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
106 logger.error("",e);
1578324d2aac backup logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1690
diff changeset
107 throw e;
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
108 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
109 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
110 };
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
111
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
112 private final Runnable syncWithChecksum = new Runnable() {
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
113 public void run() {
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
114 try {
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
115 sync(true);
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
116 } catch(ConnectException e) {
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
117 logger.error("syncWithChecksum failed: "+e.getMessage());
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
118 } catch(IOException e) {
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
119 throw new RuntimeException(e);
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
120 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
121 }
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
122 };
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
123
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
124 private void sync(boolean withChecksum) throws IOException {
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
125 List<LogFile> logs = new ArrayList<LogFile>();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
126 synchronized(this) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
127 isSyncPending = false;
1500
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
128 clearDir();
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
129 for( LogFile log : this.logs ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
130 File f = new File(dir,log.file.getName());
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
131 IoUtils.link(log.file,f);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
132 logs.add( new LogFile(f) );
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
133 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
134 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
135 List logInfo = new ArrayList();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
136 Map<String,LogFile> logMap = new HashMap<String,LogFile>();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
137 for( LogFile log : logs ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
138 Map fileInfo = new HashMap();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
139 fileInfo.put("name",log.file.getName());
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
140 fileInfo.put("end",log.end());
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
141 if( withChecksum )
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
142 fileInfo.put("checksum",log.checksum());
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
143 logInfo.add(fileInfo);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
144 logMap.put(log.file.getName(),log);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
145 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
146 for( String backupDomain : backupDomains ) {
1509
0ba144491a42 lucene.backup zip
Franklin Schmidt <fschmidt@gmail.com>
parents: 1508
diff changeset
147 RpcClient rpc = BackupServer.rpcClient(backupDomain);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
148 try {
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
149 RpcCall call = new RpcCall("login",domain,name);
1512
31b543826ca9 lucene.backup login
Franklin Schmidt <fschmidt@gmail.com>
parents: 1509
diff changeset
150 rpc.write(call);
31b543826ca9 lucene.backup login
Franklin Schmidt <fschmidt@gmail.com>
parents: 1509
diff changeset
151 rpc.read();
31b543826ca9 lucene.backup login
Franklin Schmidt <fschmidt@gmail.com>
parents: 1509
diff changeset
152 call = new RpcCall("check",logInfo);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
153 while(true) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
154 rpc.write(call);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
155 RpcResult result = rpc.read();
1512
31b543826ca9 lucene.backup login
Franklin Schmidt <fschmidt@gmail.com>
parents: 1509
diff changeset
156 //logger.info(Arrays.asList(result.returnValues).toString());
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
157 String status = (String)result.returnValues[0];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
158 if( status.equals("ok") ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
159 break;
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
160 } else if( status.equals("missing") || status.equals("bad_checksum") ) {
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
161 String fileName = (String)result.returnValues[1];
1508
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
162 if( status.equals("bad_checksum") )
86c5e7000ecf lucene.backup checksum
Franklin Schmidt <fschmidt@gmail.com>
parents: 1506
diff changeset
163 logger.error("bad_checksum "+fileName);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
164 LogFile log = logMap.get(fileName);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
165 long len = log.end() - 8;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
166 InputStream in = log.input();
1512
31b543826ca9 lucene.backup login
Franklin Schmidt <fschmidt@gmail.com>
parents: 1509
diff changeset
167 call = new RpcCall(in,len,"add",logInfo,fileName);
1675
Franklin Schmidt <fschmidt@gmail.com>
parents: 1674
diff changeset
168 logger.info("add "+fileName);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
169 } else if( status.equals("incomplete") ) {
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
170 String fileName = (String)result.returnValues[1];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
171 long logEnd = (Long)result.returnValues[2];
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
172 LogFile log = logMap.get(fileName);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
173 long len = log.end() - logEnd;
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
174 InputStream in = log.input();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
175 in.skip(logEnd-8);
1512
31b543826ca9 lucene.backup login
Franklin Schmidt <fschmidt@gmail.com>
parents: 1509
diff changeset
176 call = new RpcCall(in,len,"append",logInfo,fileName);
1675
Franklin Schmidt <fschmidt@gmail.com>
parents: 1674
diff changeset
177 // logger.info("append "+fileName);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
178 } else
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
179 throw new RuntimeException("status "+status);
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
180 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
181 } catch(RpcException e) {
1672
8dd8c556c449 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1548
diff changeset
182 logger.error("",e);
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
183 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
184 rpc.close();
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
185 }
1500
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
186 clearDir();
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
187 }
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
188
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
189 private void clearDir() throws IOException {
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
190 for( File f : dir.listFiles() ) {
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
191 IoUtils.delete(f);
Franklin Schmidt <fschmidt@gmail.com>
parents: 1499
diff changeset
192 }
1499
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
193 }
22e15cf73040 lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1488
diff changeset
194
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
195 public static BackupIndexWriter newWithRestore(LuceneIndexWriter indexWriter,File logDir,long logTime,String domain,String name)
1674
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
196 throws IOException
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
197 {
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
198 if( !logDir.exists() ) {
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
199 RpcClient rpc = BackupServer.rpcClient(backupDomains[0]);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
200 try {
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
201 RpcCall call;
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
202 RpcResult result;
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
203 call = new RpcCall("exists",domain,name);
1674
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
204 rpc.write(call);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
205 result = rpc.read();
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
206 boolean exists = (Boolean)result.returnValues[0];
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
207 if( exists ) {
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
208 logger.error("restoring "+logDir+" from backup");
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
209 File zip = File.createTempFile("luan_",".zip");
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
210 IoUtils.delete(zip);
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
211 call = new RpcCall("login",domain,name);
1674
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
212 rpc.write(call);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
213 rpc.read();
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
214 call = new RpcCall("zip");
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
215 rpc.write(call);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
216 result = rpc.read();
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
217 OutputStream out = new BufferedOutputStream(new FileOutputStream(zip));
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
218 IoUtils.copyAll(result.in,out);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
219 out.close();
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
220 IoUtils.mkdirs(logDir);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
221 String cmd = "unzip " + zip;
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
222 Process proc = Runtime.getRuntime().exec(cmd,null,logDir);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
223 IoUtils.waitFor(proc);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
224 IoUtils.delete(zip);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
225 }
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
226 } catch(RpcException e) {
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
227 throw new RuntimeException(e);
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
228 }
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
229 rpc.close();
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
230 }
1690
973d3039c421 backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents: 1675
diff changeset
231 return new BackupIndexWriter(indexWriter,logDir,logTime,domain,name);
1674
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
232 }
af18eacf187c automatic restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 1673
diff changeset
233
1488
af55cfad6e12 start lucene.backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
diff changeset
234 }