|
0
|
1 package nabble.model;
|
|
|
2
|
|
|
3 import fschmidt.db.DbDatabase;
|
|
|
4 import fschmidt.db.DbNull;
|
|
|
5 import fschmidt.util.java.DateUtils;
|
|
|
6 import fschmidt.util.java.HtmlUtils;
|
|
|
7 import fschmidt.util.mail.Mail;
|
|
|
8 import fschmidt.util.mail.MailAddress;
|
|
|
9 import fschmidt.util.mail.MailAddressException;
|
|
|
10 import fschmidt.util.mail.MailEncodingException;
|
|
|
11 import fschmidt.util.mail.MailException;
|
|
|
12 import fschmidt.util.mail.MailHome;
|
|
|
13 import fschmidt.util.mail.MailIterator;
|
|
|
14 import fschmidt.util.mail.MailParseException;
|
|
|
15 import fschmidt.util.mail.Pop3Server;
|
|
|
16 import fschmidt.util.mail.javamail.MstorInServer;
|
|
|
17 import nabble.model.lucene.HitCollector;
|
|
|
18 import nabble.model.lucene.LuceneSearcher;
|
|
|
19 import org.apache.lucene.document.Document;
|
|
|
20 import org.apache.lucene.queryParser.ParseException;
|
|
|
21 import org.apache.lucene.queryParser.QueryParser;
|
|
|
22 import org.apache.lucene.search.BooleanQuery;
|
|
|
23 import org.apache.lucene.search.Filter;
|
|
|
24 import org.apache.lucene.search.Query;
|
|
|
25 import org.apache.lucene.util.Version;
|
|
|
26 import org.slf4j.Logger;
|
|
|
27 import org.slf4j.LoggerFactory;
|
|
|
28
|
|
|
29 import java.io.File;
|
|
|
30 import java.io.IOException;
|
|
|
31 import java.io.PrintWriter;
|
|
|
32 import java.io.StringWriter;
|
|
|
33 import java.io.UnsupportedEncodingException;
|
|
|
34 import java.sql.Connection;
|
|
|
35 import java.sql.PreparedStatement;
|
|
|
36 import java.sql.ResultSet;
|
|
|
37 import java.sql.SQLException;
|
|
|
38 import java.text.DateFormat;
|
|
|
39 import java.text.SimpleDateFormat;
|
|
|
40 import java.util.ArrayList;
|
|
|
41 import java.util.Arrays;
|
|
|
42 import java.util.Collection;
|
|
|
43 import java.util.Date;
|
|
|
44 import java.util.HashSet;
|
|
|
45 import java.util.List;
|
|
|
46 import java.util.Set;
|
|
|
47 import java.util.TimeZone;
|
|
|
48 import java.util.concurrent.TimeUnit;
|
|
|
49 import java.util.regex.Matcher;
|
|
|
50 import java.util.regex.Pattern;
|
|
|
51 import javax.mail.internet.InternetAddress;
|
|
|
52 import javax.mail.internet.AddressException;
|
|
|
53
|
|
|
54
|
|
|
55 final class MailingLists {
|
|
|
56 private static final Logger logger = LoggerFactory.getLogger(MailingLists.class);
|
|
|
57
|
|
|
58 private static final float nameChangeFreq = Init.get("mlNameChangeFreq",0.1f);
|
|
|
59 static final Pop3Server pop3Server = (Pop3Server)Init.get("mailingListArchivePop3Server");
|
|
|
60
|
|
|
61 private MailingLists() {} // never
|
|
|
62
|
|
|
63 static {
|
|
|
64 if( Init.hasDaemons ) {
|
|
|
65 runMailingLists();
|
|
|
66 }
|
|
|
67 }
|
|
|
68
|
|
|
69 private static void runMailingLists() {
|
|
|
70 if( pop3Server == null ) {
|
|
|
71 logger.warn("no pop3 server defined, mailing lists not running");
|
|
|
72 return;
|
|
|
73 }
|
|
|
74 Executors.scheduleWithFixedDelay(new Runnable() {
|
|
|
75 public void run(){
|
|
|
76 try {
|
|
|
77 processMail();
|
|
|
78 processFwds();
|
|
|
79 } catch(MailException e) {
|
|
|
80 logger.error("mailing list processing",e);
|
|
|
81 }
|
|
|
82 }
|
|
|
83 }, 10, 10, TimeUnit.SECONDS );
|
|
|
84 logger.info("mailing lists enabled");
|
|
|
85 }
|
|
|
86
|
|
|
87 private static void processMail() {
|
|
|
88 MailIterator mails = pop3Server.getMail();
|
|
|
89 int count = 0;
|
|
|
90 try {
|
|
|
91 while( mails.hasNext() ) {
|
|
|
92 Mail mail = mails.next();
|
|
|
93 try {
|
|
|
94 makePost(mail);
|
|
|
95 count++;
|
|
|
96 } catch (MailAddressException e) {
|
|
|
97 logger.warn("mail:\n"+mail.getRawInput(),e); // screwed-up mail
|
|
|
98 } catch (Exception e) {
|
|
|
99 logger.error("mail:\n"+mail.getRawInput(),e);
|
|
|
100 }
|
|
|
101 }
|
|
|
102 } finally {
|
|
|
103 mails.close();
|
|
|
104 if( count > 0 )
|
|
|
105 logger.error("Processed " + count + " emails.");
|
|
|
106 }
|
|
|
107 }
|
|
|
108
|
|
|
109 static MailingList.ImportResult importMbox(File file,MailingListImpl ml,String mailErrorsToS,int maxErrors)
|
|
|
110 throws ModelException
|
|
|
111 {
|
|
|
112 final DateFormat mailmanDateFormat = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy");
|
|
|
113 final DateFormat mailDateFormat = new javax.mail.internet.MailDateFormat();
|
|
|
114 mailmanDateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
|
|
|
115 mailDateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
|
|
|
116 MailAddress mailErrorsTo = new MailAddress(mailErrorsToS);
|
|
|
117 MstorInServer server = new MstorInServer(file);
|
|
|
118 server.setMetaEnabled(false);
|
|
|
119 MailIterator mails = server.getMail();
|
|
|
120 try {
|
|
|
121 int imported = 0;
|
|
|
122 int errors = 0;
|
|
|
123 while( mails.hasNext() ) {
|
|
|
124 Mail mail = mails.next();
|
|
|
125 try {
|
|
|
126 try {
|
|
|
127 mail.getFrom();
|
|
|
128 } catch (MailAddressException e) {
|
|
|
129 String[] from = mail.getHeader("From");
|
|
|
130 if (from == null || from.length == 0)
|
|
|
131 throw new MailAddressException("'From' not found in the header", e);
|
|
|
132 mail.setHeader("From", from[0].replace(" at ", "@"));
|
|
|
133 }
|
|
|
134 Date sentDate = mail.getSentDate();
|
|
|
135 if ((sentDate==null || sentDate.getTime() < 0) && mail.getHeader("Date")!=null) {
|
|
|
136 String dateH = mail.getHeader("Date")[0];
|
|
|
137 if (dateH!=null) {
|
|
|
138 try {
|
|
|
139 sentDate = mailmanDateFormat.parse(dateH);
|
|
|
140 } catch (java.text.ParseException e) {}
|
|
|
141 if (sentDate!=null)
|
|
|
142 mail.setSentDate(sentDate);
|
|
|
143 }
|
|
|
144 }
|
|
|
145 if ((sentDate==null || sentDate.getTime() < 0) && mail.getHeader("Resent-date")!=null) {
|
|
|
146 String dateH = mail.getHeader("Resent-date")[0];
|
|
|
147 if (dateH!=null) {
|
|
|
148 try {
|
|
|
149 sentDate = mailDateFormat.parse(dateH);
|
|
|
150 } catch (java.text.ParseException e) {}
|
|
|
151 if (sentDate!=null)
|
|
|
152 mail.setSentDate(sentDate);
|
|
|
153 }
|
|
|
154 }
|
|
|
155 if ((sentDate==null || sentDate.getTime() < 0)) {
|
|
|
156 String rawInput = mail.getRawInput();
|
|
|
157 try {
|
|
|
158 String dateH = rawInput.substring(rawInput.indexOf(' ',5), rawInput.indexOf('\n')).trim();
|
|
|
159 sentDate = mailmanDateFormat.parse(dateH);
|
|
|
160 } catch (Exception e) {
|
|
|
161 logger.error("",e); // what kind of exception is ok?
|
|
|
162 }
|
|
|
163 if (sentDate!=null)
|
|
|
164 mail.setSentDate(sentDate);
|
|
|
165 }
|
|
|
166 makeForumPost(mail,ml,true);
|
|
|
167 imported++;
|
|
|
168 } catch (Exception e) {
|
|
|
169 sendErrorMail(mail, e, mailErrorsTo);
|
|
|
170 errors++;
|
|
|
171 if( errors >= maxErrors )
|
|
|
172 throw ModelException.newInstance("import_mbox_errors",""+errors+" errors reached after importing "+imported+" messages");
|
|
|
173 }
|
|
|
174 }
|
|
|
175 final int imported2 = imported;
|
|
|
176 final int errors2 = errors;
|
|
|
177 return new MailingList.ImportResult() {
|
|
|
178 public int getImported() { return imported2; }
|
|
|
179 public int getErrors() { return errors2; }
|
|
|
180 };
|
|
|
181 } finally {
|
|
|
182 mails.close();
|
|
|
183 }
|
|
|
184 }
|
|
|
185
|
|
|
186 private static void makePost(Mail mail)
|
|
|
187 throws ModelException
|
|
|
188 {
|
|
|
189 MailingListImpl ml = getMailingList(mail);
|
|
|
190 if (ml == null) {
|
|
|
191 logger.info("Mailing list not found for: " + Arrays.asList(mail.getTo()));
|
|
|
192 return;
|
|
|
193 }
|
|
|
194 if (checkForward(mail, ml)) {
|
|
|
195 return;
|
|
|
196 }
|
|
|
197 if (checkPending(mail, ml)) {
|
|
|
198 return;
|
|
|
199 }
|
|
|
200 makeForumPost(mail, ml, false);
|
|
|
201 }
|
|
|
202
|
|
|
203 private static void makeForumPost(Mail mail, MailingListImpl ml, boolean isImport)
|
|
|
204 throws ModelException
|
|
|
205 {
|
|
|
206 String messageID = getMessageID(mail, msgFmt);
|
|
|
207 mail.setMessageID(messageID);
|
|
|
208
|
|
|
209 String message = mail.getRawInput();
|
|
|
210 message = message.replace("\000",""); // postgres can't handle 0
|
|
|
211 if( !msgFmt.isOk(message) )
|
|
|
212 return;
|
|
|
213 String text = msgFmt.getMailText(message,null);
|
|
|
214 NodeImpl forum = ml.getForumImpl();
|
|
|
215
|
|
|
216 if( doNotArchive(text) || (doNotArchive(mail) && !ml.ignoreNoArchive()) ) {
|
|
|
217 logger.info("XNoArchive in "+forum.getSubject());
|
|
|
218 return;
|
|
|
219 }
|
|
|
220
|
|
|
221 DbDatabase db;
|
|
|
222 try {
|
|
|
223 db = forum.siteKey.getDb();
|
|
|
224 } catch(UpdatingException e) {
|
|
|
225 return; // hack for schema migration
|
|
|
226 }
|
|
|
227 db.beginTransaction();
|
|
|
228 try {
|
|
|
229 forum = (NodeImpl)forum.getGoodCopy();
|
|
|
230 MailingListImpl mailingList = forum.getMailingListImpl();
|
|
|
231
|
|
|
232 {
|
|
|
233 NodeImpl post = forum.getNodeImplFromMessageID(messageID);
|
|
|
234 if( post != null) {
|
|
|
235 if(isImport)
|
|
|
236 return;
|
|
|
237 throw new RuntimeException("MessageID "+messageID+" already in db for forum "+forum.getId());
|
|
|
238 }
|
|
|
239 }
|
|
|
240
|
|
|
241 UserImpl user = getUser(mail, mailingList);
|
|
|
242 if (user.isNoArchive())
|
|
|
243 return;
|
|
|
244
|
|
|
245 String subject = mailingList.fixSubject(mail.getSubject());
|
|
|
246 if( subject==null || subject.trim().equals("") )
|
|
|
247 subject = "(no subject)";
|
|
|
248
|
|
|
249 if (!isImport) {
|
|
|
250 ListServer oldListServer = mailingList.getListServer();
|
|
|
251 if (oldListServer==ListServer.unknown || oldListServer instanceof ListServer.Mailman) {
|
|
|
252 ListServer listServer = detectListServer(mail);
|
|
|
253 if (listServer!=null && listServer!=oldListServer && (oldListServer==ListServer.unknown || listServer==ListServer.mailman21)) {
|
|
|
254 mailingList.setListServer(listServer);
|
|
|
255 mailingList.update();
|
|
|
256 }
|
|
|
257 }
|
|
|
258 }
|
|
|
259
|
|
|
260 Date now = new Date();
|
|
|
261 Date date = mail.getSentDate();
|
|
|
262 if( date==null || date.compareTo(now) > 0 || date.getTime() < 0)
|
|
|
263 date = now;
|
|
|
264
|
|
|
265 boolean isGuessedParent = false;
|
|
|
266 String parentID = getParentID(mail, messageID);
|
|
|
267 NodeImpl parent = forum.getNodeImplFromMessageID(parentID);
|
|
|
268 if ( parent!=null && threadBySubject(forum, subject, parent.getSubject()) ) {
|
|
|
269 parent = null;
|
|
|
270 }
|
|
|
271
|
|
|
272 NodeImpl[] orphans = NodeImpl.getFromParentID(messageID,mailingList);
|
|
|
273 if ( parent==null ) {
|
|
|
274 try {
|
|
|
275 parent = guessParent(mail, date, mailingList, subject, orphans);
|
|
|
276 if ( parent != null )
|
|
|
277 isGuessedParent = true;
|
|
|
278 } catch(IOException e) {
|
|
|
279 logger.error("guessParent failed",e);
|
|
|
280 }
|
|
|
281 }
|
|
|
282
|
|
|
283 NodeImpl post = NodeImpl.newChildNode(Node.Kind.POST,user,subject,message,msgFmt,parent==null?forum:parent);
|
|
|
284 if( parent==null && parentID != null ) {
|
|
|
285 logger.debug("Orphan "+messageID+" starting new thread ");
|
|
|
286 isGuessedParent = true;
|
|
|
287 }
|
|
|
288
|
|
|
289 post.setWhenCreated(date);
|
|
|
290 post.setMessageID(messageID);
|
|
|
291 if (isGuessedParent) {
|
|
|
292 post.setGuessedParent(parentID);
|
|
|
293 } else if (parent==null) {
|
|
|
294 // for root posts which do not have parentID set guess flag to uncertain
|
|
|
295 post.setGuessedParent((Boolean) null);
|
|
|
296 }
|
|
|
297 post.insert(false);
|
|
|
298 if( isGuessedParent && parentID==null )
|
|
|
299 logger.debug("no parentID for "+post);
|
|
|
300
|
|
|
301 for (NodeImpl orphan : orphans) {
|
|
|
302 try {
|
|
|
303 if (!threadBySubject(forum, subject, orphan.getSubject())) {
|
|
|
304 orphan.changeParentImpl(post);
|
|
|
305 }
|
|
|
306 } catch (ModelException.NodeLoop e) {
|
|
|
307 logger.error("", e); // should not happen now...
|
|
|
308 orphan.getDbRecord().fields().put("parent_message_id", DbNull.STRING);
|
|
|
309 orphan.getDbRecord().update();
|
|
|
310 }
|
|
|
311 }
|
|
|
312
|
|
|
313 db.commitTransaction();
|
|
|
314 } finally {
|
|
|
315 db.endTransaction();
|
|
|
316 }
|
|
|
317 }
|
|
|
318
|
|
|
319 private static boolean threadBySubject(Node forum, String subject, String parentSubject) {
|
|
|
320 if (!forumsThreadedBySubject.contains(forum.getId())) return false;
|
|
|
321 return ! normalizeSubject(subject).equals(normalizeSubject(parentSubject));
|
|
|
322 }
|
|
|
323
|
|
|
324 static final Set<Long> forumsThreadedBySubject = new HashSet<Long>(Arrays.asList((Long[])Init.get("forumsThreadedBySubject", new Long[0])));
|
|
|
325
|
|
|
326
|
|
|
327 private static void sendErrorMail(Mail mail, Exception e, MailAddress mailTo) {
|
|
|
328 if( e instanceof UnsupportedEncodingException
|
|
|
329 || e instanceof MailAddressException
|
|
|
330 || e instanceof MailEncodingException
|
|
|
331 || e instanceof MailParseException
|
|
|
332 ) {
|
|
|
333 logger.info(e.toString());
|
|
|
334 } else {
|
|
|
335 logger.error("",e);
|
|
|
336 }
|
|
|
337 StringWriter sb = new StringWriter();
|
|
|
338 PrintWriter out = new PrintWriter(sb);
|
|
|
339 e.printStackTrace( out );
|
|
|
340 out.close();
|
|
|
341 String msg = e.getMessage();
|
|
|
342 if (msg!=null && msg.indexOf('\n')>=0) msg = msg.substring(0, msg.indexOf('\n')).trim();
|
|
|
343 String subject = "error: "+msg;
|
|
|
344 MailSubsystem.sendErrorMail(mail, mailTo, subject, sb.toString());
|
|
|
345 }
|
|
|
346
|
|
|
347 private static String getParentID(Mail mail, String messageID) {
|
|
|
348 String[] inReplyTos = mail.getHeader("In-Reply-To");
|
|
|
349 if( inReplyTos == null ) {
|
|
|
350 inReplyTos = mail.getHeader("In-Reply-to");
|
|
|
351 if( inReplyTos != null )
|
|
|
352 logger.error("does this happen - case sensitive");
|
|
|
353 }
|
|
|
354 if( inReplyTos != null ) {
|
|
|
355 for (String inReplyTo : inReplyTos) {
|
|
|
356 for( String s : MailSubsystem.stripMultiBrackets(inReplyTo) ) {
|
|
|
357 if (!s.equals(messageID) && !s.equals("")) return s;
|
|
|
358 }
|
|
|
359 }
|
|
|
360 }
|
|
|
361 try {
|
|
|
362 String[] refs = mail.getHeader("References");
|
|
|
363 if( refs != null ) {
|
|
|
364 for (String ref : refs) {
|
|
|
365 List<String> list = MailSubsystem.stripMultiBrackets(ref);
|
|
|
366 if( list.isEmpty() )
|
|
|
367 continue;
|
|
|
368 String s = list.get(list.size()-1);
|
|
|
369 if (!s.equals(messageID) && !s.equals("")) return s;
|
|
|
370 }
|
|
|
371 }
|
|
|
372 } catch(MailParseException e) {
|
|
|
373 logger.warn("screwed up References for messageID="+messageID,e);
|
|
|
374 }
|
|
|
375 return null;
|
|
|
376 }
|
|
|
377
|
|
|
378 private static NodeImpl guessParent(Mail mail, Date date, MailingListImpl mailingList, String subject, NodeImpl[] orphans) throws IOException {
|
|
|
379 Set<NodeImpl> offspring = new HashSet<NodeImpl>();
|
|
|
380 for (NodeImpl orphan : orphans) {
|
|
|
381 for (NodeImpl n : orphan.getDescendantImpls()) {
|
|
|
382 offspring.add(n);
|
|
|
383 }
|
|
|
384 }
|
|
|
385 return guessParent(mail, date, mailingList, subject, offspring);
|
|
|
386 }
|
|
|
387
|
|
|
388 static NodeImpl guessParent(NodeImpl post, Collection<NodeImpl> ignore) {
|
|
|
389 Mail mail = MailHome.newMail(post.getMessage().getRaw());
|
|
|
390 NodeImpl forum = post.getAppImpl();
|
|
|
391 if (forum == null) {
|
|
|
392 return null; // detached post
|
|
|
393 }
|
|
|
394 MailingListImpl mailingList = forum.getAssociatedMailingListImpl();
|
|
|
395 if (mailingList == null) {
|
|
|
396 return null; // forum no longer a mailing list
|
|
|
397 }
|
|
|
398 try {
|
|
|
399 Set<NodeImpl> ignoreSet = new HashSet<NodeImpl>();
|
|
|
400 for( NodeImpl n : post.getDescendantImpls() ) {
|
|
|
401 ignoreSet.add(n);
|
|
|
402 }
|
|
|
403 if (ignore != null) {
|
|
|
404 ignoreSet.addAll(ignore);
|
|
|
405 }
|
|
|
406 return guessParent(mail, post.getWhenCreated(), mailingList, post.getSubject(), ignoreSet);
|
|
|
407 } catch (IOException e) {
|
|
|
408 throw new RuntimeException(e);
|
|
|
409 }
|
|
|
410 }
|
|
|
411
|
|
|
412 private static NodeImpl guessParent(Mail mail, Date date, MailingListImpl mailingList, String subject, Set<NodeImpl> offspring)
|
|
|
413 throws IOException {
|
|
|
414 // attach to ancestor if any
|
|
|
415 NodeImpl forum = mailingList.getForumImpl();
|
|
|
416 try {
|
|
|
417 String[] refs = mail.getHeader("References");
|
|
|
418 if( refs != null ) {
|
|
|
419 for( String ref : refs ) {
|
|
|
420 final List<String> list = MailSubsystem.stripMultiBrackets(ref);
|
|
|
421 for( int i=list.size()-1; i>=0; i-- ) {
|
|
|
422 String ancestorID = list.get(i);
|
|
|
423 NodeImpl parent = forum.getNodeImplFromMessageID(ancestorID);
|
|
|
424 if (parent!=null && !offspring.contains(parent) && !threadBySubject(forum,subject,parent.getSubject())) {
|
|
|
425 logger.debug("Attaching orphan "+mail.getMessageID()+" to grandparent "+parent);
|
|
|
426 return parent;
|
|
|
427 }
|
|
|
428 }
|
|
|
429 }
|
|
|
430 }
|
|
|
431 } catch(MailParseException e) {
|
|
|
432 logger.warn("screwed up References",e);
|
|
|
433 }
|
|
|
434 // handle lost In-Reply-To headers
|
|
|
435 // heuristics - use Thread-Topic header to find matching subjects in last 3 days
|
|
|
436 String[] threadTopics = mail.getHeader("Thread-Topic");
|
|
|
437 String threadTopic = threadTopics==null?null:mailingList.fixSubject(threadTopics[0]);
|
|
|
438 long forumId = forum.getId();
|
|
|
439 Filter filter = Lucene.getRangeFilter(DateUtils.addDays(date, -7), date);
|
|
|
440 SiteImpl site = forum.getSiteImpl();
|
|
|
441 LuceneSearcher searcher = Lucene.newSearcher(site);
|
|
|
442 try {
|
|
|
443 if( threadTopic != null ) {
|
|
|
444 threadTopic = threadTopic.toLowerCase();
|
|
|
445 if (threadTopic.startsWith("re: "))
|
|
|
446 threadTopic = threadTopic.substring(4);
|
|
|
447 threadTopic = threadTopic.trim();
|
|
|
448 if (!threadTopic.equals("")) {
|
|
|
449 NodeImpl parent = (NodeImpl)getPriorPost(site,searcher, forumId, threadTopic, filter, date, offspring);
|
|
|
450 if( parent!=null && !offspring.contains(parent) && !threadBySubject(forum,subject,parent.getSubject())) return parent;
|
|
|
451 }
|
|
|
452 }
|
|
|
453 // if no thread-topic, but subject starts with Re:, try with subject
|
|
|
454 subject = subject.toLowerCase();
|
|
|
455 if( subject.startsWith("re: ") ) {
|
|
|
456 subject = subject.substring(4).trim();
|
|
|
457 if ( !subject.equals(threadTopic) && !"".equals(subject) ) {
|
|
|
458 NodeImpl parent = (NodeImpl)getPriorPost(site,searcher, forumId, subject, filter, date, offspring);
|
|
|
459 if( parent!=null && !offspring.contains(parent)) return parent;
|
|
|
460 }
|
|
|
461 }
|
|
|
462 } finally {
|
|
|
463 searcher.close();
|
|
|
464 }
|
|
|
465 return null;
|
|
|
466 }
|
|
|
467
|
|
|
468 private static boolean checkPending(Mail mail, MailingListImpl ml) {
|
|
|
469 if (checkPending(getMessageID(mail, msgFmt), ml)) {
|
|
|
470 return true;
|
|
|
471 }
|
|
|
472 String[] xMessageId = mail.getHeader("X-Message-Id");
|
|
|
473 if (xMessageId != null && xMessageId.length > 0 && checkPending(MailSubsystem.stripBrackets(xMessageId[0]), ml)) {
|
|
|
474 return true;
|
|
|
475 }
|
|
|
476 xMessageId = mail.getHeader("X-Original-Message-Id");
|
|
|
477 if (xMessageId != null && xMessageId.length > 0 && checkPending(MailSubsystem.stripBrackets(xMessageId[0]), ml)) {
|
|
|
478 return true;
|
|
|
479 }
|
|
|
480 return false;
|
|
|
481 }
|
|
|
482
|
|
|
483 private static boolean checkPending(String messageID, MailingListImpl ml) {
|
|
|
484 NodeImpl pendingPost = ml.getForumImpl().getNodeImplFromMessageID(messageID);
|
|
|
485 if( pendingPost==null )
|
|
|
486 return false;
|
|
|
487 Node.MailToList mail = pendingPost.getMailToList();
|
|
|
488 if( mail == null ) {
|
|
|
489 logger.warn("MessageID "+messageID+" already in db as "+pendingPost+" for forum "+ml.getId());
|
|
|
490 } else if( !mail.isPending() ) {
|
|
|
491 logger.error("post not pending "+pendingPost);
|
|
|
492 } else {
|
|
|
493 mail.clearPending();
|
|
|
494 }
|
|
|
495 return true;
|
|
|
496 }
|
|
|
497
|
|
|
498 private static boolean doNotArchive(Mail mail) {
|
|
|
499 String[] xNoArchive = mail.getHeader("X-No-Archive");
|
|
|
500 if (xNoArchive != null && xNoArchive.length > 0 && "yes".equalsIgnoreCase(xNoArchive[0])) {
|
|
|
501 return true;
|
|
|
502 }
|
|
|
503 String[] xArchive = mail.getHeader("X-Archive");
|
|
|
504 if (xArchive != null && xArchive.length > 0 && xArchive[0] != null && (xArchive[0].startsWith("expiry") || "no".equalsIgnoreCase(xArchive[0]))) {
|
|
|
505 return true;
|
|
|
506 }
|
|
|
507 String[] archive = mail.getHeader("Archive");
|
|
|
508 if (archive != null && archive.length > 0 && "no".equalsIgnoreCase(archive[0])) {
|
|
|
509 return true;
|
|
|
510 }
|
|
|
511 return false;
|
|
|
512 }
|
|
|
513
|
|
|
514 private static final Pattern xNoArchivePtn = Pattern.compile("(?im)\\AX-No-Archive: yes *$");
|
|
|
515 private static boolean doNotArchive(String text) {
|
|
|
516 return xNoArchivePtn.matcher(text).find();
|
|
|
517 }
|
|
|
518
|
|
|
519 private static MailingListImpl getMailingList(Mail mail) {
|
|
|
520 MailingListImpl ml = null;
|
|
|
521 String[] a = mail.getHeader("Envelope-To");
|
|
|
522 if (a == null)
|
|
|
523 a = mail.getHeader("X-Delivered-to"); // fastmail
|
|
|
524 if (a == null)
|
|
|
525 a = mail.getHeader("X-Original-To"); // postfix
|
|
|
526 if( a.length > 1 )
|
|
|
527 a = new String[] { a[0] };
|
|
|
528 for( String address : a[0].split(",") ) {
|
|
|
529 address = address.trim();
|
|
|
530 MailingListImpl candidate = MailingListImpl.getMailingListByEnvelopeAddress(address);
|
|
|
531 if (candidate == null) {
|
|
|
532 // escaped list mail, bounce mail
|
|
|
533 String returnPath = MailSubsystem.getReturnPath(mail);
|
|
|
534 if( returnPath.equals(address) )
|
|
|
535 continue; // ignore spam
|
|
|
536 MailSubsystem.bounce(mail,
|
|
|
537 "Delivery to the following recipient failed permanently:\n\n "
|
|
|
538 + address
|
|
|
539 + "\n\nNo archive exists for this address.\n"
|
|
|
540 );
|
|
|
541 logger.warn( "no mailing list found for "+address+" - bouncing mail to "+returnPath + ":\n" + mail);
|
|
|
542 } else {
|
|
|
543 if( ml != null )
|
|
|
544 logger.error("mailing list already set");
|
|
|
545 ml = candidate;
|
|
|
546 }
|
|
|
547 }
|
|
|
548 return ml;
|
|
|
549 }
|
|
|
550
|
|
|
551 private static String extractDomain(String email) {
|
|
|
552 String domain = email.substring(email.indexOf('@')+1).toLowerCase();
|
|
|
553 // hack to unify google messages
|
|
|
554 return domain.replace("google.com","googlegroups.com");
|
|
|
555 }
|
|
|
556
|
|
|
557 private static boolean checkForward(Mail mail, MailingListImpl ml) {
|
|
|
558 // check if the archive guessed from subscription address is not presented in the
|
|
|
559 // common headers that contain list address, forward to the archive owner in this case
|
|
|
560 String envTo[] = mail.getHeader("Envelope-To");
|
|
|
561 if (envTo == null)
|
|
|
562 envTo = mail.getHeader("X-Delivered-to"); // fastmail
|
|
|
563 if (envTo == null)
|
|
|
564 envTo = mail.getHeader("X-Original-To"); // postfix
|
|
|
565 String originalTo = envTo[0];
|
|
|
566 {
|
|
|
567 MailAddress[] to = mail.getTo();
|
|
|
568 if( to==null || to.length!=1 || !to[0].getAddrSpec().equalsIgnoreCase(originalTo) )
|
|
|
569 return false;
|
|
|
570 }
|
|
|
571 // check for domain of the message's From: or Reply-To:
|
|
|
572 String listAddress = ml.getListAddress();
|
|
|
573 String domain = extractDomain(listAddress);
|
|
|
574 String maintenanceMessageReplyTo = null;
|
|
|
575 {
|
|
|
576 MailAddress[] replyTos = mail.getReplyTo();
|
|
|
577 if (replyTos != null) {
|
|
|
578 for (MailAddress replyTo : replyTos) {
|
|
|
579 String replyDomain = extractDomain(replyTo.getAddrSpec());
|
|
|
580 if (replyDomain.endsWith(domain) || domain.endsWith(replyDomain)) {
|
|
|
581 maintenanceMessageReplyTo = replyTo.getAddrSpec();
|
|
|
582 break;
|
|
|
583 }
|
|
|
584 }
|
|
|
585 }
|
|
|
586 }
|
|
|
587 MailAddress from = mail.getFrom();
|
|
|
588 // first we compare the domains
|
|
|
589 if( maintenanceMessageReplyTo == null && from != null && (extractDomain(from.getAddrSpec()).endsWith(domain) || domain.endsWith(extractDomain(from.getAddrSpec()))))
|
|
|
590 maintenanceMessageReplyTo = from.getAddrSpec();
|
|
|
591 // check if this is a majordomo email
|
|
|
592 if (maintenanceMessageReplyTo == null && from != null && from.getAddrSpec().toLowerCase().startsWith("majordomo@"))
|
|
|
593 maintenanceMessageReplyTo = from.getAddrSpec();
|
|
|
594
|
|
|
595 if( maintenanceMessageReplyTo != null ) {
|
|
|
596 mail.setReplyTo( new MailAddress(fwdEmail(originalTo, maintenanceMessageReplyTo) ));
|
|
|
597 MailAddress ownerAddress = getArchiveOwnerAddress(ml);
|
|
|
598 MailHome.getDefaultSmtpServer().send(mail,ownerAddress);
|
|
|
599 logger.info("Forwarding maintenance message to owner: " + ownerAddress);
|
|
|
600 logger.info(mail.getRawInput());
|
|
|
601 return true;
|
|
|
602 }
|
|
|
603 if( MailSubsystem.getReturnPath(mail).equals("") ) {
|
|
|
604 MailAddress ownerAddress = getArchiveOwnerAddress(ml);
|
|
|
605 MailHome.getDefaultSmtpServer().send(mail,ownerAddress);
|
|
|
606 logger.info("Forwarding maintenance message to owner: " + ownerAddress);
|
|
|
607 logger.info(mail.getRawInput());
|
|
|
608 return true;
|
|
|
609 }
|
|
|
610 logger.info("Bouncing email to: " + MailSubsystem.getReturnPath(mail) + " / envelopeTo = " + originalTo + "\n" + mail.getRawInput());
|
|
|
611 MailSubsystem.bounce(mail,
|
|
|
612 "Delivery to the following recipient failed permanently:\n\n "
|
|
|
613 + originalTo
|
|
|
614 + "\n\nThis email address is only for archiving mailing lists and should not be used directly.\n"
|
|
|
615 );
|
|
|
616 return true;
|
|
|
617 }
|
|
|
618
|
|
|
619 private static MailAddress getArchiveOwnerAddress(MailingListImpl mailingList) {
|
|
|
620 // If this list was exported to another server, we have to send this email
|
|
|
621 // to the person that did the export. Otherwise we send to the current owner.
|
|
|
622 String exportOwner = mailingList.getExportOwner();
|
|
|
623 if (exportOwner == null) {
|
|
|
624 // Send to the current owner...
|
|
|
625 User owner = mailingList.getForumImpl().getOwnerImpl();
|
|
|
626 return new MailAddress(owner.getEmail(), owner.getName());
|
|
|
627 } else {
|
|
|
628 // Send to the person who exported the archive...
|
|
|
629 return new MailAddress(exportOwner);
|
|
|
630 }
|
|
|
631 }
|
|
|
632
|
|
|
633 private static MailAddress toMailAddress(String s) {
|
|
|
634 try {
|
|
|
635 InternetAddress ia = new InternetAddress(s);
|
|
|
636 return new MailAddress(ia.getAddress(),ia.getPersonal());
|
|
|
637 } catch(AddressException e) {
|
|
|
638 return null;
|
|
|
639 }
|
|
|
640 }
|
|
|
641
|
|
|
642 private static UserImpl getUser(Mail mail, MailingListImpl mailingList) {
|
|
|
643 MailAddress addr = null;
|
|
|
644 String a[] = mail.getHeader("X-Original-From");
|
|
|
645 if( a != null )
|
|
|
646 addr = toMailAddress(a[0]);
|
|
|
647 if( addr == null )
|
|
|
648 addr = mail.getFrom();
|
|
|
649 String email = addr.getAddrSpec();
|
|
|
650 if (email == null || "".equals(email.trim()))
|
|
|
651 {
|
|
|
652 throw new MailAddressException("Invalid sender address: "+addr);
|
|
|
653 }
|
|
|
654 SiteImpl site = mailingList.getForumImpl().getSiteImpl();
|
|
|
655 UserImpl user = site.getUserImplFromEmail(email);
|
|
|
656 if( user==null || !user.isRegistered() ) {
|
|
|
657 String username;
|
|
|
658 if( email.equalsIgnoreCase(mailingList.getListAddress()) ) {
|
|
|
659 username = mailingList.getForum().getSubject() + " mailing list";
|
|
|
660 } else {
|
|
|
661 username = addr.getDisplayName();
|
|
|
662 if( username == null || "".equals(username.trim()) ) {
|
|
|
663 username = email.indexOf('@')>0 ? email.substring(0, email.indexOf('@')) : email;
|
|
|
664 }
|
|
|
665 }
|
|
|
666 if( username.endsWith(" (JIRA)") ) {
|
|
|
667 username = "JIRA "+email;
|
|
|
668 }
|
|
|
669 if( user==null ) {
|
|
|
670 user = UserImpl.createGhost(site,email);
|
|
|
671 user.setNameLike(username,false);
|
|
|
672 user.insert();
|
|
|
673 } else {
|
|
|
674 String oldName = user.getName();
|
|
|
675 if( !oldName.toLowerCase().startsWith(username.toLowerCase())
|
|
|
676 && (Math.random() < nameChangeFreq)
|
|
|
677 ) {
|
|
|
678 user.setNameLike(username,false);
|
|
|
679 user.getDbRecord().update();
|
|
|
680 logger.warn("changed name of "+user+" from '"+oldName+"' to '"+user.getName()+"'");
|
|
|
681 }
|
|
|
682 }
|
|
|
683 }
|
|
|
684 return user;
|
|
|
685 }
|
|
|
686
|
|
|
687 static final MailMessageFormat msgFmt = new MailMessageFormat('m', "mail");
|
|
|
688
|
|
|
689 private static Node getPriorPost(final SiteImpl site,final LuceneSearcher searcher, long forumId, final String subject, Filter filter, final Date to, final Set offspring) throws IOException {
|
|
|
690 //String phrase = "\""+QueryParser.escape(subject.replace('\"',' '))+"\"";
|
|
|
691 try {
|
|
|
692 NodeSearcher.Builder query = new NodeSearcher.Builder(site,forumId);
|
|
|
693 query.addNodeKind(Node.Kind.POST);
|
|
|
694 QueryParser parser = new QueryParser(Version.LUCENE_CURRENT,Lucene.SUBJECT_FLD, Lucene.analyzer);
|
|
|
695 parser.setDefaultOperator(QueryParser.AND_OPERATOR);
|
|
|
696 Query subjectQuery = parser.parse(QueryParser.escape(subject.replace('\"',' ').replace("&&"," ")));
|
|
|
697 if (! (subjectQuery instanceof BooleanQuery && ((BooleanQuery)subjectQuery).getClauses().length==0) )
|
|
|
698 query.addQuery(subjectQuery);
|
|
|
699 final Node[] resultHolder = new Node[1];
|
|
|
700 searcher.search( query.build().getQuery(), filter, new HitCollector() {
|
|
|
701 protected void process(Document doc) {
|
|
|
702 NodeImpl post = Lucene.node(site,doc);
|
|
|
703 if (post==null)
|
|
|
704 return;
|
|
|
705 String parentSubject = post.getSubject().toLowerCase();
|
|
|
706 if ( (parentSubject.equals(subject) || (parentSubject.startsWith("re: ") && parentSubject.substring(4).trim().equals(subject)))
|
|
|
707 && to.after(post.getWhenCreated())
|
|
|
708 && (resultHolder[0]==null || resultHolder[0].getWhenCreated().before(post.getWhenCreated()))
|
|
|
709 && !offspring.contains(post)
|
|
|
710 )
|
|
|
711 resultHolder[0] = post;
|
|
|
712 }
|
|
|
713 });
|
|
|
714 Node result = resultHolder[0];
|
|
|
715
|
|
|
716 if (result != null) {
|
|
|
717 // find the uppermost post with almost-the-same subject
|
|
|
718 String subjectEtalon = normalizeSubject(subject.toLowerCase());
|
|
|
719 Node resultCandidate = result.getTopic();
|
|
|
720 while (result != null) {
|
|
|
721 String resultSubject = normalizeSubject(result.getSubject().toLowerCase());
|
|
|
722 if (!resultSubject.equals(subjectEtalon)) break; // break when subject really changes
|
|
|
723
|
|
|
724 // this post has almost-the-same subject
|
|
|
725 if (!offspring.contains(result)) {
|
|
|
726 // set only if this node is not presented in escape-set
|
|
|
727 resultCandidate = result;
|
|
|
728 }
|
|
|
729 result = result.getParent();
|
|
|
730 }
|
|
|
731 result = resultCandidate;
|
|
|
732 }
|
|
|
733 return result;
|
|
|
734 } catch (ParseException e) {
|
|
|
735 throw new RuntimeException(e);
|
|
|
736 }
|
|
|
737 }
|
|
|
738
|
|
|
739 private static final Pattern bracketRegex = Pattern.compile("\\[[^\\[]+\\]");
|
|
|
740
|
|
|
741 static Pattern prefixRegex(String prefixes) {
|
|
|
742 return Pattern.compile( "^((" + prefixes + "): *)+" );
|
|
|
743 }
|
|
|
744
|
|
|
745 private static final Pattern defaultPrefixRegex = prefixRegex("re|aw|res|fwd|答复");
|
|
|
746
|
|
|
747 /**
|
|
|
748 * Remove from subject all possible prefixes which are added while forwarding, replying, etc...
|
|
|
749 *
|
|
|
750 * @param subject original subject
|
|
|
751 * @return normalized subject
|
|
|
752 */
|
|
|
753 private static String normalizeSubject(String subject) {
|
|
|
754 return normalizeSubject(subject,defaultPrefixRegex);
|
|
|
755 }
|
|
|
756
|
|
|
757 static String normalizeSubject(String subject,Pattern prefixRegex) {
|
|
|
758 if (subject != null) {
|
|
|
759 subject = subject.toLowerCase().trim();
|
|
|
760 subject = bracketRegex.matcher(subject).replaceAll("");
|
|
|
761 subject = prefixRegex.matcher(subject).replaceAll("");
|
|
|
762 }
|
|
|
763 return subject;
|
|
|
764 }
|
|
|
765
|
|
|
766 static void nop() {}
|
|
|
767
|
|
|
768
|
|
|
769 private static ListServer detectListServer(Mail mail) {
|
|
|
770 String[] mailman = mail.getHeader("X-Mailman-Version");
|
|
|
771 if (mailman!=null && mailman.length==1 && mailman[0]!=null) {
|
|
|
772 if (mailman[0].startsWith("2.0")) {
|
|
|
773 return ListServer.mailman20;
|
|
|
774 } else if (mailman[0].startsWith("2.1")) {
|
|
|
775 return ListServer.mailman21;
|
|
|
776 } else if (mailman[0].startsWith("2.")) {
|
|
|
777 logger.error("unknown mailman version: "+mailman[0]+" in message "+mail.getMessageID());
|
|
|
778 }
|
|
|
779 return null;
|
|
|
780 }
|
|
|
781
|
|
|
782 String[] mList = mail.getHeader("Mailing-List");
|
|
|
783 if (mList!=null && mList.length==1 && mList[0]!=null) {
|
|
|
784 if (mList[0].indexOf("run by ezmlm")>=0) {
|
|
|
785 return ListServer.ezmlm;
|
|
|
786 } else if (mList[0].indexOf("@yahoogr")>0 || mList[0].indexOf("@gruposyahoo")>0) {
|
|
|
787 return ListServer.yahoo;
|
|
|
788 } else if (mList[0].indexOf("@googlegroups")>0) {
|
|
|
789 return ListServer.google;
|
|
|
790 }
|
|
|
791 }
|
|
|
792
|
|
|
793 String[] listproc = mail.getHeader("X-Listprocessor-Version");
|
|
|
794 if (listproc!=null && listproc.length==1 && listproc[0]!=null) {
|
|
|
795 if (listproc[0].indexOf("ListProc")>=0) {
|
|
|
796 if (listproc[0].indexOf("CREN")>=0) {
|
|
|
797 return ListServer.listproc;
|
|
|
798 } else {
|
|
|
799 return ListServer.oldlistproc;
|
|
|
800 }
|
|
|
801 } else {
|
|
|
802 logger.error("unknown listproc version: "+listproc[0]+" in message "+mail.getMessageID());
|
|
|
803 return null;
|
|
|
804 }
|
|
|
805 }
|
|
|
806
|
|
|
807 String[] ecartis = mail.getHeader("X-ecartis-version");
|
|
|
808 if (ecartis!=null && ecartis.length==1 && ecartis[0]!=null) {
|
|
|
809 if (ecartis[0].indexOf("Ecartis")>=0) {
|
|
|
810 return ListServer.ecartis;
|
|
|
811 } else {
|
|
|
812 logger.error("unknown ecartis version: "+ecartis[0]+" in message "+mail.getMessageID());
|
|
|
813 return null;
|
|
|
814 }
|
|
|
815 }
|
|
|
816
|
|
|
817 String[] lyris = mail.getHeader("X-LISTMANAGER-Message-Id");
|
|
|
818 if (lyris!=null && lyris.length==1 && lyris[0]!=null) {
|
|
|
819 if (lyris[0].indexOf("LISTMANAGER")>=0) {
|
|
|
820 return ListServer.lyris;
|
|
|
821 } else {
|
|
|
822 logger.error("unexpected x-listmanager-message-id header: "+lyris[0]+" in message "+mail.getMessageID());
|
|
|
823 return null;
|
|
|
824 }
|
|
|
825 }
|
|
|
826
|
|
|
827 String[] xListServer = mail.getHeader("X-ListServer");
|
|
|
828 if (xListServer!=null && xListServer.length==1 && xListServer[0]!=null) {
|
|
|
829 if (xListServer[0].indexOf("CommuniGate")>=0) {
|
|
|
830 return ListServer.communigate;
|
|
|
831 } else {
|
|
|
832 logger.error("unknown x-listserver header: "+xListServer[0]+" in message "+mail.getMessageID());
|
|
|
833 return null;
|
|
|
834 }
|
|
|
835 }
|
|
|
836
|
|
|
837 // may not be reliable
|
|
|
838 String[] listSubscribe = mail.getHeader("List-Subscribe");
|
|
|
839 if (listSubscribe!=null && listSubscribe.length==1 && listSubscribe[0]!=null) {
|
|
|
840 if (listSubscribe[0].indexOf("+subscribe@")>0) {
|
|
|
841 return ListServer.mlmmj;
|
|
|
842 } else if (listSubscribe[0].indexOf("listserver@")>=0) {
|
|
|
843 return ListServer.listserver;
|
|
|
844 } else if (listSubscribe[0].indexOf("sympa@")>=0) {
|
|
|
845 return ListServer.sympa;
|
|
|
846 }
|
|
|
847 }
|
|
|
848
|
|
|
849 // not possible to detect:
|
|
|
850 // listserv
|
|
|
851 // majordomo / majordomo2
|
|
|
852 // smartlist
|
|
|
853
|
|
|
854 return null;
|
|
|
855 }
|
|
|
856 /*
|
|
|
857 static void redoGuessedParents() {}
|
|
|
858
|
|
|
859 public static void rethreadPosts(boolean inBatch) throws SQLException{
|
|
|
860 rethreadPosts(1, 0, inBatch);
|
|
|
861 }
|
|
|
862
|
|
|
863 public static void rethreadPosts(long startingPostId, boolean inBatch) throws SQLException{
|
|
|
864 rethreadPosts(startingPostId, 0, inBatch);
|
|
|
865 }
|
|
|
866
|
|
|
867 static void rethreadPosts(long startingPostId, long forumId, boolean inBatch) throws SQLException{
|
|
|
868 Logger batchLog = inBatch ? Batch.logger : logger;
|
|
|
869 if(startingPostId > 1)
|
|
|
870 batchLog.info("Starting rethread from post " + startingPostId);
|
|
|
871 if(forumId > 0)
|
|
|
872 batchLog.info("Rethreading for forum " + forumId);
|
|
|
873
|
|
|
874 //the WHERE condition is post_id > postId !
|
|
|
875 long postId = startingPostId - 1;
|
|
|
876 int processed_post_count = 0;
|
|
|
877 int modified_post_count = 0;
|
|
|
878 int null_parent_count = 0;
|
|
|
879 NodeImpl post = null;
|
|
|
880 boolean more = true;
|
|
|
881 try {
|
|
|
882 outer: while (more) {
|
|
|
883 Connection con = Db.db.getConnection();
|
|
|
884 try {
|
|
|
885 con.setAutoCommit(false);
|
|
|
886 PreparedStatement stmt = con.prepareStatement(
|
|
|
887 (forumId > 0) ?
|
|
|
888 "SELECT * FROM descendants(" + forumId + ") WHERE node_id > ? AND " +
|
|
|
889 "(guessed_parent='t' OR guessed_parent is null) AND " +
|
|
|
890 "msg_fmt='m'" +
|
|
|
891 "ORDER BY node_id LIMIT 1"
|
|
|
892 :
|
|
|
893 "SELECT * FROM node WHERE node_id > ? AND " +
|
|
|
894 "(guessed_parent='t' OR guessed_parent is null) AND " +
|
|
|
895 "msg_fmt='m'" +
|
|
|
896 "ORDER BY node_id LIMIT 1"
|
|
|
897 );
|
|
|
898 stmt.setLong(1, postId);
|
|
|
899 ResultSet rs = stmt.executeQuery();
|
|
|
900 more = false;
|
|
|
901 while (rs.next()) {
|
|
|
902 more = true;
|
|
|
903 post = NodeImpl.getNode(rs);
|
|
|
904 try {
|
|
|
905 postId = post.getId();
|
|
|
906 Mail mail = MailHome.newMail(post.getMessage().getRaw());
|
|
|
907 MailingListImpl mailingList = post.getAppImpl().getAssociatedMailingListImpl();
|
|
|
908 if (mailingList==null) continue; // forum no longer a mailing list
|
|
|
909 List<NodeImpl> descendants = new ArrayList<NodeImpl>();
|
|
|
910 for( NodeImpl n : post.getDescendantImpls() ) {
|
|
|
911 descendants.add(n);
|
|
|
912 }
|
|
|
913 NodeImpl parent = guessParent(mail, post.getWhenCreated(), mailingList, post.getSubject(), descendants.toArray(new NodeImpl[0]));
|
|
|
914
|
|
|
915 if (parent != null && parent.getId() != post.getParentId()) {
|
|
|
916 try {
|
|
|
917 batchLog.debug("setting parent of " + post + " to " + parent);
|
|
|
918 post.setGuessedParent(parent);
|
|
|
919 if(++ modified_post_count % 100 == 0)
|
|
|
920 batchLog.info("Modified " + modified_post_count + " posts");
|
|
|
921 } catch (ModelException.NodeLoop e) {
|
|
|
922 batchLog.error("",e);
|
|
|
923 }
|
|
|
924 } else if(parent == null && post.getParentId() != 0 && post.getParent().getKind()!=Node.Kind.APP){
|
|
|
925 batchLog.info("Null parent at " + post);
|
|
|
926 null_parent_count ++;
|
|
|
927 }
|
|
|
928 } catch(Exception x){
|
|
|
929 batchLog.error("Exception at post " + post, x);
|
|
|
930 break outer;
|
|
|
931 }
|
|
|
932
|
|
|
933 if(++ processed_post_count % 3000 == 0)
|
|
|
934 batchLog.info("Processed " + processed_post_count + " posts, current postId: "+postId);
|
|
|
935
|
|
|
936 if (inBatch) {
|
|
|
937 Batch.checkStopped();
|
|
|
938 }
|
|
|
939 }
|
|
|
940 stmt.close();
|
|
|
941 con.commit();
|
|
|
942 } finally {
|
|
|
943 con.close();
|
|
|
944 }
|
|
|
945 }
|
|
|
946 } finally {
|
|
|
947 batchLog.info("Exited at post " + post);
|
|
|
948 batchLog.info("Processed " + processed_post_count + " posts");
|
|
|
949 batchLog.info("Modified " + modified_post_count + " posts");
|
|
|
950 batchLog.info("Guessed null parent at " + null_parent_count + " posts");
|
|
|
951 }
|
|
|
952 }
|
|
|
953 */
|
|
|
954 private static void getRethreadIds(Connection con,long parentId,Collection<Long> ids)
|
|
|
955 throws SQLException
|
|
|
956 {
|
|
|
957 PreparedStatement stmt = con.prepareStatement(
|
|
|
958 "select node_id, guessed_parent, msg_fmt from node where parent_id = ?"
|
|
|
959 );
|
|
|
960 stmt.setLong(1,parentId);
|
|
|
961 ResultSet rs = stmt.executeQuery();
|
|
|
962 while( rs.next() ) {
|
|
|
963 long id = rs.getLong("node_id");
|
|
|
964 if( "m".equals(rs.getString("msg_fmt"))
|
|
|
965 && ( rs.getBoolean("guessed_parent") || rs.wasNull() )
|
|
|
966 )
|
|
|
967 ids.add(id);
|
|
|
968 getRethreadIds(con,id,ids);
|
|
|
969 }
|
|
|
970 rs.close();
|
|
|
971 stmt.close();
|
|
|
972 }
|
|
|
973
|
|
|
974 static void rethreadForum(NodeImpl forum, boolean inBatch) throws SQLException{
|
|
|
975 long forumId = forum.getId();
|
|
|
976 long rethreadStart = System.currentTimeMillis();
|
|
|
977 Logger batchLog = inBatch ? Batch.logger : logger;
|
|
|
978 batchLog.info("Rethreading for forum " + forumId);
|
|
|
979 SiteKey siteKey = forum.getSiteImpl().siteKey;
|
|
|
980 DbDatabase db = siteKey.getDb();
|
|
|
981
|
|
|
982 Collection<Long> ids = new ArrayList<Long>();
|
|
|
983 {
|
|
|
984 Connection con = db.getConnection();
|
|
|
985 long queryStart = System.currentTimeMillis();
|
|
|
986 getRethreadIds(con,forumId,ids);
|
|
|
987 batchLog.info("Query took " + (System.currentTimeMillis() - queryStart) + " ms");
|
|
|
988 con.close();
|
|
|
989 }
|
|
|
990
|
|
|
991 batchLog.info(ids.size() + " posts to process...");
|
|
|
992
|
|
|
993 //the WHERE condition is post_id > postId !
|
|
|
994 int processed_post_count = 0;
|
|
|
995 int modified_post_count = 0;
|
|
|
996 int null_parent_count = 0;
|
|
|
997
|
|
|
998 try {
|
|
|
999 while (ids.size() > 0) {
|
|
|
1000 Connection con = db.getConnection();
|
|
|
1001 try {
|
|
|
1002 con.setAutoCommit(false);
|
|
|
1003
|
|
|
1004 long id = ids.iterator().next();
|
|
|
1005 ids.remove(id);
|
|
|
1006 NodeImpl post = NodeImpl.getNode(siteKey,id);
|
|
|
1007 try {
|
|
|
1008 Mail mail = MailHome.newMail(post.getMessage().getRaw());
|
|
|
1009 MailingListImpl mailingList = post.getAppImpl().getAssociatedMailingListImpl();
|
|
|
1010 if (mailingList==null) continue; // forum no longer a mailing list
|
|
|
1011 List<NodeImpl> descendants = new ArrayList<NodeImpl>();
|
|
|
1012 for( NodeImpl n : post.getDescendantImpls() ) {
|
|
|
1013 descendants.add(n);
|
|
|
1014 }
|
|
|
1015 NodeImpl parent = guessParent(mail, post.getWhenCreated(), mailingList, post.getSubject(), descendants.toArray(new NodeImpl[0]));
|
|
|
1016
|
|
|
1017 if (parent != null && parent.getId() != post.getParentId()) {
|
|
|
1018 try {
|
|
|
1019 batchLog.info("setting parent of " + post + " to " + parent);
|
|
|
1020 post.setGuessedParent(parent);
|
|
|
1021
|
|
|
1022 if(++ modified_post_count % 1000 == 0)
|
|
|
1023 batchLog.info("Modified " + modified_post_count + " posts");
|
|
|
1024 } catch (ModelException.NodeLoop e) {
|
|
|
1025 batchLog.error("",e);
|
|
|
1026 }
|
|
|
1027 } else if(parent == null && post.getParentId() != 0 && post.getParent().getKind()!=Node.Kind.APP){
|
|
|
1028 batchLog.info("Null parent at " + post);
|
|
|
1029 null_parent_count ++;
|
|
|
1030 }
|
|
|
1031 } catch(Exception x){
|
|
|
1032 batchLog.error("Exception at " + post + " - message:\n"+post.getMessage().getRaw(), x);
|
|
|
1033 break;
|
|
|
1034 }
|
|
|
1035
|
|
|
1036 if(++ processed_post_count % 1000 == 0)
|
|
|
1037 batchLog.info("Processed " + processed_post_count + " posts, current postId: "+id);
|
|
|
1038
|
|
|
1039 if (inBatch) {
|
|
|
1040 Batch.checkStopped();
|
|
|
1041 }
|
|
|
1042 con.commit();
|
|
|
1043 } finally {
|
|
|
1044 con.close();
|
|
|
1045 }
|
|
|
1046 }
|
|
|
1047 } finally {
|
|
|
1048 batchLog.info("Rethread took " + (System.currentTimeMillis() - rethreadStart) + " ms");
|
|
|
1049 batchLog.info("Processed " + processed_post_count + " posts");
|
|
|
1050 batchLog.info("Modified " + modified_post_count + " posts");
|
|
|
1051 batchLog.info("Guessed null parent at " + null_parent_count + " posts");
|
|
|
1052 }
|
|
|
1053 }
|
|
|
1054
|
|
|
1055
|
|
|
1056 /**
|
|
|
1057 * Get or create message id from an email
|
|
|
1058 *
|
|
|
1059 * @param mail email message
|
|
|
1060 * @param msgFmt message format to use
|
|
|
1061 * @return message id, never null
|
|
|
1062 */
|
|
|
1063 private static String getMessageID(Mail mail, Message.Format msgFmt) {
|
|
|
1064 String[] messageIds = mail.getHeader("Message-Id"); // returns both Id and ID
|
|
|
1065 if (messageIds == null || messageIds.length == 0 || messageIds[messageIds.length - 1] == null) {
|
|
|
1066 return calcMessageID(mail, msgFmt);
|
|
|
1067 } else {
|
|
|
1068 return MailSubsystem.stripBrackets(messageIds[messageIds.length - 1]);
|
|
|
1069 }
|
|
|
1070 }
|
|
|
1071
|
|
|
1072 /**
|
|
|
1073 * Create a new message if for an email message
|
|
|
1074 *
|
|
|
1075 * @param mail mail message to process
|
|
|
1076 * @param msgFmt message format to use
|
|
|
1077 * @return a new message id, never null
|
|
|
1078 */
|
|
|
1079 private static String calcMessageID(Mail mail, Message.Format msgFmt) {
|
|
|
1080 StringBuilder msgId = new StringBuilder();
|
|
|
1081 msgId.append("MissingID.");
|
|
|
1082 String text = msgFmt.getText(mail.getRawInput(),null);
|
|
|
1083 msgId.append(Integer.toHexString(text.hashCode()));
|
|
|
1084 MailAddress from = mail.getFrom();
|
|
|
1085 if (from != null) msgId.append(Integer.toHexString(from.toString().hashCode()));
|
|
|
1086 MailAddress[] to = mail.getTo();
|
|
|
1087 if (to != null && to.length > 0) msgId.append(Integer.toHexString(to[0].toString().hashCode()));
|
|
|
1088 Date date = mail.getSentDate();
|
|
|
1089 if (date != null) msgId.append(Integer.toHexString(date.hashCode()));
|
|
|
1090 String subject = mail.getSubject();
|
|
|
1091 if (subject != null) msgId.append(Integer.toHexString(subject.hashCode()));
|
|
|
1092 msgId.append("@nabble.com");
|
|
|
1093 return msgId.toString();
|
|
|
1094 }
|
|
|
1095
|
|
|
1096
|
|
|
1097
|
|
|
1098
|
|
|
1099
|
|
|
1100 private static final Pop3Server fwdPop3Server = (Pop3Server)Init.get("fwdPop3Server");
|
|
|
1101
|
|
|
1102 private static class Lazy {
|
|
|
1103 static final String emailPrefix;
|
|
|
1104 static final String emailSuffix;
|
|
|
1105 static final Pattern pattern;
|
|
|
1106 static {
|
|
|
1107 String addrSpec = fwdPop3Server.getUsername();
|
|
|
1108 int ind = addrSpec.indexOf('@');
|
|
|
1109 emailPrefix = addrSpec.substring(0, ind) + "+";
|
|
|
1110 emailSuffix = addrSpec.substring(ind);
|
|
|
1111 pattern = Pattern.compile(
|
|
|
1112 Pattern.quote(emailPrefix) + "([^@]+)\\+([^@]+)\\+([^@]+)" + Pattern.quote(emailSuffix)
|
|
|
1113 , Pattern.CASE_INSENSITIVE
|
|
|
1114 );
|
|
|
1115 }
|
|
|
1116 }
|
|
|
1117
|
|
|
1118 private static void processFwds() {
|
|
|
1119 if( fwdPop3Server == null ) {
|
|
|
1120 logger.error("fwdPop3Server not defined");
|
|
|
1121 System.exit(-1);
|
|
|
1122 }
|
|
|
1123 MailIterator mails = fwdPop3Server.getMail();
|
|
|
1124 try {
|
|
|
1125 while( mails.hasNext() ) {
|
|
|
1126 Mail mail = mails.next();
|
|
|
1127 try {
|
|
|
1128 fwdMail(mail);
|
|
|
1129 } catch (Exception e) {
|
|
|
1130 logger.error("mail:\n"+mail.getRawInput(),e);
|
|
|
1131 }
|
|
|
1132 }
|
|
|
1133 } finally {
|
|
|
1134 mails.close();
|
|
|
1135 }
|
|
|
1136 }
|
|
|
1137
|
|
|
1138 private static void fwdMail(Mail mail) {
|
|
|
1139 String[] envTo = mail.getHeader("Envelope-To");
|
|
|
1140 if (envTo == null)
|
|
|
1141 envTo = mail.getHeader("X-Delivered-to"); // fastmail
|
|
|
1142 if (envTo == null)
|
|
|
1143 envTo = mail.getHeader("X-Original-To"); // postfix
|
|
|
1144 String originalTo = envTo[0];
|
|
|
1145 Matcher matcher = Lazy.pattern.matcher(originalTo);
|
|
|
1146 if( !matcher.matches() )
|
|
|
1147 throw new RuntimeException("invalid email: "+originalTo);
|
|
|
1148 String fwdFrom = emailDecode(matcher.group(1));
|
|
|
1149 String fwdTo = emailDecode(matcher.group(2));
|
|
|
1150 if( (fwdFrom+fwdTo).hashCode() != Integer.parseInt(matcher.group(3)) )
|
|
|
1151 throw new RuntimeException("invalid hash: "+originalTo);
|
|
|
1152 mail.setFrom(new MailAddress(fwdFrom));
|
|
|
1153 mail.setTo(new MailAddress(fwdTo));
|
|
|
1154 logger.info("Forwarding email to mailing list: " + fwdTo);
|
|
|
1155 MailHome.getDefaultSmtpServer().send(mail);
|
|
|
1156 }
|
|
|
1157
|
|
|
1158 private static String fwdEmail(String from,String to) {
|
|
|
1159 return Lazy.emailPrefix + emailEncode(from) + '+' + emailEncode(to) + '+' + (from+to).hashCode() + Lazy.emailSuffix;
|
|
|
1160 }
|
|
|
1161
|
|
|
1162 private static String emailEncode(String s) {
|
|
|
1163 return HtmlUtils.urlEncode(s).replace('%','~');
|
|
|
1164 }
|
|
|
1165
|
|
|
1166 private static String emailDecode(String s) {
|
|
|
1167 return HtmlUtils.urlDecode(s.replace('~','%'));
|
|
|
1168 }
|
|
|
1169
|
|
|
1170 }
|
