repo/luan: src/luan/modules/lucene/LuceneIndex.java annotate

annotate src/luan/modules/lucene/LuceneIndex.java @ 1687:f48db13ae2d9

unlogged lucene support

author	Franklin Schmidt <fschmidt@gmail.com>
date	Thu, 09 Jun 2022 19:44:41 -0600
parents	46cf5137cb6b
children	973d3039c421

rev	line source
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1 package luan.modules.lucene;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	2
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	3 import java.io.Closeable;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	4 import java.io.File;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	6 import java.io.FileInputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	7 import java.io.IOException;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	8 import java.lang.ref.Reference;
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	9 import java.lang.ref.WeakReference;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	10 import java.sql.SQLException;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	11 import java.util.Arrays;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	12 import java.util.Iterator;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	13 import java.util.Map;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	14 import java.util.HashMap;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	15 import java.util.List;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	16 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	17 import java.util.Set;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	18 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	19 import java.util.Collections;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	20 import java.util.concurrent.atomic.AtomicInteger;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	21 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	22 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	23 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	24 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	25 import org.apache.lucene.analysis.Analyzer;
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	26 import org.apache.lucene.analysis.TokenStream;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	27 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	28 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	29 import org.apache.lucene.document.Document;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	30 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	31 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	32 import org.apache.lucene.document.StringField;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	33 import org.apache.lucene.document.TextField;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	34 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	35 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	36 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	37 import org.apache.lucene.index.IndexableField;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	38 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	39 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	40 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	41 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	42 import org.apache.lucene.index.SnapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	43 import org.apache.lucene.index.IndexCommit;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	44 import org.apache.lucene.index.AtomicReaderContext;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	45 import org.apache.lucene.index.CheckIndex;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	46 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	47 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	48 import org.apache.lucene.util.Version;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	49 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	50 import org.apache.lucene.util.NumericUtils;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	51 import org.apache.lucene.search.Query;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	52 import org.apache.lucene.search.PrefixQuery;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	53 import org.apache.lucene.search.TermQuery;
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	54 import org.apache.lucene.search.MatchAllDocsQuery;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	55 import org.apache.lucene.search.TopDocs;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	56 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	57 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	58 import org.apache.lucene.search.IndexSearcher;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	59 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	60 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	61 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	62 import org.apache.lucene.search.Scorer;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	63 import org.apache.lucene.search.Explanation;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	64 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	65 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	66 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	67 import org.apache.lucene.search.highlight.Fragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	68 import org.apache.lucene.search.highlight.NullFragmenter;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	69 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	70 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	71 import org.apache.lucene.search.highlight.TokenGroup;
1474 13cbce740e1e LowercaseAnalyzer Franklin Schmidt <fschmidt@gmail.com> parents: 1459 diff changeset	72 import goodjava.lucene.analysis.LowercaseAnalyzer;
1459 b04b8fc5f4f4 GoodQueryParser Franklin Schmidt <fschmidt@gmail.com> parents: 1458 diff changeset	73 import goodjava.lucene.queryparser.GoodQueryParser;
1458 6b6c11c9164e goodjava.lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1454 diff changeset	74 import goodjava.lucene.queryparser.FieldParser;
6b6c11c9164e goodjava.lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1454 diff changeset	75 import goodjava.lucene.queryparser.MultiFieldParser;
6b6c11c9164e goodjava.lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1454 diff changeset	76 import goodjava.lucene.queryparser.StringFieldParser;
6b6c11c9164e goodjava.lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1454 diff changeset	77 import goodjava.lucene.queryparser.NumberFieldParser;
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	78 import goodjava.lucene.api.GoodIndexWriter;
3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	79 import goodjava.lucene.api.LuceneIndexWriter;
3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	80 import goodjava.lucene.api.GoodIndexWriterConfig;
3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	81 import goodjava.lucene.api.LuceneUtils;
1538 634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	82 import goodjava.lucene.logging.LoggingIndexWriter;
1551 9cc4cee39b8b add LuanOpDoer Franklin Schmidt <fschmidt@gmail.com> parents: 1548 diff changeset	83 import goodjava.lucene.logging.OpDoer;
1672 8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	84 import goodjava.lucene.backup.BackupIndexWriter;
1402 27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	85 import goodjava.parser.ParseException;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	86 import luan.modules.Utils;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	87 import luan.Luan;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	88 import luan.LuanTable;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	89 import luan.LuanFunction;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	90 import luan.LuanException;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	91 import luan.LuanRuntimeException;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	92 import luan.modules.parsers.LuanToString;
1402 27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	93 import goodjava.logging.Logger;
27efb1fcbcb5 move luan.lib to goodjava Franklin Schmidt <fschmidt@gmail.com> parents: 1399 diff changeset	94 import goodjava.logging.LoggerFactory;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	95
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	96
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	97 public final class LuceneIndex {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	98 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	99
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	100 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	101
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	102 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	103 throws LuanException, IOException, ClassNotFoundException, SQLException
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	104 {
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	105 String key = indexDir.getCanonicalPath();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	106 synchronized(indexes) {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	107 Reference<LuceneIndex> ref = indexes.get(key);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	108 if( ref != null ) {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	109 LuceneIndex li = ref.get();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	110 if( li != null ) {
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	111 Object version = options.get(luan,"version");
1676 daa4214fa91a minor Franklin Schmidt <fschmidt@gmail.com> parents: 1674 diff changeset	112 if( (version==null \|\| version.equals(li.version)) && li.indexDir.exists() )
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	113 return li;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	114 li.closeWriter();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	115 }
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	116 }
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	117 LuceneIndex li = new LuceneIndex(luan,indexDir,options);
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	118 indexes.put(key, new WeakReference<LuceneIndex>(li));
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	119 return li;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	120 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	121 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	122
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	123 private static final Version luceneVersion = Version.LUCENE_4_9;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	124 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1532 060ff7695317 Lucene.type.lowercase Franklin Schmidt <fschmidt@gmail.com> parents: 1530 diff changeset	125 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion));
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	126 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion));
1538 634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	127 private static final SortField ID_SORT = new SortField("id",SortField.Type.LONG);
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	128 private static final SortField ID_DESC_SORT = new SortField("id",SortField.Type.LONG,true);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	129
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	130 private final Object version;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	131
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	132 private final ReentrantLock writeLock = new ReentrantLock();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	133 private final File indexDir;
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	134 private GoodIndexWriter writer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	135 private DirectoryReader reader;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	136 private IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	137 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	138 private final MultiFieldParser mfp;
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	139 private final Analyzer analyzer; // ???
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	140
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	141 private FSDirectory fsDir;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	142 private int writeCount;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	143 private AtomicInteger writeCounter = new AtomicInteger();
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	144 private final GoodIndexWriterConfig config;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	145
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	146 private final PostgresBackup postgresBackup;
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	147 private boolean wasCreated;
1538 634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	148 private final File logDir;
1548 736ec76bbf42 lucene log work Franklin Schmidt <fschmidt@gmail.com> parents: 1544 diff changeset	149 private final long logTime;
1672 8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	150 private final String name;
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	151 private final String domain;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	152
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	153 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	154 throws LuanException, IOException, ClassNotFoundException, SQLException
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	155 {
1420 225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	156 options = new LuanTable(options);
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	157 this.version = options.remove("version");
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	158 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type");
225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	159 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	160 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
1420 225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	161 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	162 LuanFunction supplementer = Utils.removeFunction(options,"supplementer");
1538 634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	163 logDir = (File)options.remove("log_dir");
1548 736ec76bbf42 lucene log work Franklin Schmidt <fschmidt@gmail.com> parents: 1544 diff changeset	164 logTime = (Long)options.remove("log_time");
1672 8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	165 name = (String)options.remove("name");
1420 225808b90cee options handling Franklin Schmidt <fschmidt@gmail.com> parents: 1410 diff changeset	166 Utils.checkEmpty(options);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	167
1672 8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	168 {
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	169 LuanTable module = (LuanTable)luan.require("luan:http/Http.luan");
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	170 String domain = (String)module.get(luan,"domain");
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	171 if( domain == null )
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	172 domain = "localhost";
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	173 this.domain = domain;
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	174 }
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	175
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	176 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	177 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	178 mfp.fields.put( "id", NumberFieldParser.LONG );
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	179 this.indexDir = indexDir;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	180 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	181 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	182 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	183 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	184 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	185 this.analyzer = analyzer;
1685 46cf5137cb6b misc fixes Franklin Schmidt <fschmidt@gmail.com> parents: 1676 diff changeset	186 this.config = new SupplementingConfig(luceneVersion,mfp,supplementer);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	187 wasCreated = reopen();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	188 if( postgresSpec == null ) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	189 postgresBackup = null;
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	190 } else {
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	191 postgresBackup = new PostgresBackup(postgresSpec);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	192 if( !wasCreated && postgresBackup.wasCreated ) {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	193 logger.error("rebuilding postgres backup");
1453 928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	194 rebuild_postgres_backup(luan);
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	195 /*
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	196 } else if( wasCreated && !postgresBackup.wasCreated ) {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	197 logger.error("restoring from postgres");
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	198 restore_from_postgres();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	199 */
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	200 }
179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	201 }
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	202 }
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	203
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	204 public boolean reopen() throws IOException {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	205 fsDir = FSDirectory.open(indexDir);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	206 boolean wasCreated = !fsDir.getDirectory().exists();
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	207 writer = new LuceneIndexWriter(fsDir,config);
1672 8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	208 if( logDir != null ) {
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	209 if( BackupIndexWriter.backupDomains == null ) {
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	210 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime);
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	211 } else {
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	212 String name = this.domain;
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	213 if( this.name != null )
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	214 name += "~" + this.name;
1674 af18eacf187c automatic restore Franklin Schmidt <fschmidt@gmail.com> parents: 1673 diff changeset	215 writer = BackupIndexWriter.newWithRestore((LuceneIndexWriter)writer,logDir,logTime,name);
1672 8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	216 }
8dd8c556c449 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1623 diff changeset	217 }
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	218 reader = DirectoryReader.open(fsDir);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	219 searcher = new IndexSearcher(reader);
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	220 initId();
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	221 return wasCreated;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	222 }
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	223
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	224 private void wrote() {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	225 writeCounter.incrementAndGet();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	226 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	227
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	228 public void delete_all() throws IOException, SQLException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	229 boolean commit = !writeLock.isHeldByCurrentThread();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	230 writeLock.lock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	231 try {
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	232 writer.deleteAll();
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	233 id = 0;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	234 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	235 postgresBackup.deleteAll();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	236 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	237 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	238 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	239 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	240 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	241 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	242
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	243 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	244 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	245 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	246 return new Term(key,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	247 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	248
1543 1db694d98003 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1541 diff changeset	249 private static final Set<String> ID_SET = Collections.singleton("id");
1db694d98003 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1541 diff changeset	250
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	251 private void backupDelete(Query query)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	252 throws IOException, SQLException, LuanException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	253 {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	254 if( postgresBackup != null ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	255 final List<Long> ids = new ArrayList<Long>();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	256 IndexSearcher searcher = openSearcher();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	257 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	258 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	259 @Override public void collect(int iDoc) throws IOException {
1543 1db694d98003 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1541 diff changeset	260 Document doc = searcher.doc( docBase + iDoc, ID_SET );
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	261 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	262 ids.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	263 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	264 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	265 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	266 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	267 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	268 }
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	269 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	270 for( Long id : ids ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	271 postgresBackup.delete(id);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	272 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	273 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	274 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	275 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	276
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	277 public void delete(String queryStr)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	278 throws IOException, ParseException, SQLException, LuanException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	279 {
1459 b04b8fc5f4f4 GoodQueryParser Franklin Schmidt <fschmidt@gmail.com> parents: 1458 diff changeset	280 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	281
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	282 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	283 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	284 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	285 backupDelete(query);
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	286 writer.deleteDocuments(query);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	287 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	288 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	289 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	290 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	291 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	292 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	293
1535 e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	294 public void reindex(String queryStr)
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	295 throws IOException, ParseException
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	296 {
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	297 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	298
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	299 boolean commit = !writeLock.isHeldByCurrentThread();
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	300 writeLock.lock();
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	301 try {
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	302 writer.reindexDocuments("id",query);
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	303 if(commit) writer.commit();
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	304 } finally {
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	305 wrote();
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	306 writeLock.unlock();
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	307 }
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	308 }
e73b72a510b4 Lucene reindex Franklin Schmidt <fschmidt@gmail.com> parents: 1533 diff changeset	309
1687 f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	310 public void save( Luan luan, LuanTable doc, LuanTable unstored, Map<String,Float> boosts )
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	311 throws LuanException, IOException, SQLException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	312 {
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	313 Object obj = doc.get(luan,"id");
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	314 Long id;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	315 try {
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	316 id = (Long)obj;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	317 } catch(ClassCastException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	318 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	319 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	320
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	321 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	322 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	323 try {
1687 f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	324 if( unstored!=null \|\| boosts!=null ) {
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	325 if( unstored == null )
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	326 throw new LuanException("unstored required with boosts");
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	327 if( boosts == null )
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	328 throw new LuanException("boosts required with unstored");
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	329 if( id != null )
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	330 throw new LuanException("update not supported");
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	331 if( postgresBackup != null )
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	332 throw new LuanException("not supported with postgres backup");
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	333 if( !(writer instanceof LuceneIndexWriter) )
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	334 throw new LuanException("not supported with index logging");
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	335 id = ++this.id;
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	336 doc.put(luan,"id",id);
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	337 LuceneIndexWriter liw = (LuceneIndexWriter)writer;
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	338 liw.addDocument( toLucene(doc), toLucene(unstored), boosts );
f48db13ae2d9 unlogged lucene support Franklin Schmidt <fschmidt@gmail.com> parents: 1685 diff changeset	339 } else if( id == null ) {
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	340 id = ++this.id;
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	341 doc.put(luan,"id",id);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	342 if( postgresBackup != null )
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	343 postgresBackup.add(luan,doc);
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	344 writer.addDocument(toLucene(doc));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	345 } else {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	346 if( postgresBackup != null )
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	347 postgresBackup.update(luan,doc);
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	348 writer.updateDocument( "id", toLucene(doc) );
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	349 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	350 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	351 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	352 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	353 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	354 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	355 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	356
1533 28f06ac6ce39 add not_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1532 diff changeset	357 public boolean is_in_transaction() {
28f06ac6ce39 add not_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1532 diff changeset	358 return writeLock.isHeldByCurrentThread();
28f06ac6ce39 add not_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1532 diff changeset	359 }
28f06ac6ce39 add not_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1532 diff changeset	360
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	361 public Object run_in_transaction(Luan luan,LuanFunction fn)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	362 throws IOException, LuanException, SQLException
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	363 {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	364 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	365 writeLock.lock();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	366 boolean ok = false;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	367 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	368 if( commit && postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	369 postgresBackup.begin();
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	370 Object rtn = fn.call(luan);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	371 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	372 if(commit) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	373 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	374 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	375 writer.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	376 }
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	377 return rtn;
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	378 } finally {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	379 if( !ok && commit ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	380 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	381 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	382 writer.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	383 reopen();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	384 }
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	385 wrote();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	386 writeLock.unlock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	387 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	388 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	389
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	390 // ???
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	391 public Object run_in_lock(Luan luan,LuanFunction fn) throws IOException, LuanException {
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	392 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	393 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	394 writeLock.lock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	395 try {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	396 synchronized(this) {
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	397 return fn.call(luan);
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	398 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	399 } finally {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	400 wrote();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	401 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	402 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	403 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	404
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	405
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	406 private long id;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	407
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	408 private void initId() throws IOException {
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	409 TopDocs td = searcher.search(new MatchAllDocsQuery(),1,new Sort(ID_DESC_SORT));
c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	410 switch(td.scoreDocs.length) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	411 case 0:
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	412 id = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	413 break;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	414 case 1:
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	415 id = (Long)searcher.doc(td.scoreDocs[0].doc).getField("id").numericValue();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	416 break;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	417 default:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	418 throw new RuntimeException();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	419 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	420 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	421
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	422 /*
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	423 public void backup(String zipFile) throws LuanException, IOException {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	424 if( !zipFile.endsWith(".zip") )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	425 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	426 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	427 try {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	428 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	429 for( String fileName : ic.getFileNames() ) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	430 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	431 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	432 Utils.copyAll(in,out);
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	433 in.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	434 out.closeEntry();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	435 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	436 out.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	437 } finally {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	438 snapshotDeletionPolicy.release(ic);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	439 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	440 }
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	441 */
1129 3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	442 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	443 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy();
1129 3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	444 }
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	445
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	446 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException {
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	447 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy();
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	448 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	449 try {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	450 String dir = fsDir.getDirectory().toString();
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	451 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames()));
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	452 return fn.call(luan,dir,fileNames);
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	453 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	454 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	455 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	456 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	457
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	458 public void tag(String tag) throws IOException {
1556 52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	459 boolean commit = !writeLock.isHeldByCurrentThread();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	460 writeLock.lock();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	461 try {
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	462 writer.tag(tag);
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	463 if(commit) writer.commit();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	464 } finally {
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	465 writeLock.unlock();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	466 }
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	467 }
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	468
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	469
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	470
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	471 public String to_string() {
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	472 return writer.getLuceneIndexWriter().getDirectory().toString();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	473 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	474
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	475 protected void finalize() throws Throwable {
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	476 close();
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	477 super.finalize();
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	478 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	479
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	480 public void close() throws IOException, SQLException {
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	481 closeWriter();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	482 reader.close();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	483 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	484
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	485 private void closeWriter() throws IOException, SQLException {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	486 writeLock.lock();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	487 try {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	488 writer.close();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	489 if( postgresBackup != null )
67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	490 postgresBackup.close();
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	491 } finally {
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	492 writeLock.unlock();
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	493 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	494 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	495
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	496
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	497 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	498 final IndexSearcher searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	499 final Query query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	500 int docID;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	501
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	502 DocFn(IndexSearcher searcher,Query query) {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	503 this.searcher = searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	504 this.query = query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	505 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	506
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	507 @Override public Object call(Luan luan,Object[] args) throws LuanException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	508 try {
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	509 LuanTable doc = toTable(searcher.doc(docID));
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	510 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	511 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	512 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	513 } else {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	514 return doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	515 }
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	516 } catch(IOException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	517 throw new LuanException(e);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	518 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	519 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	520 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	521
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	522 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	523 int docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	524 int i = 0;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	525
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	526 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	527 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	528 this.docBase = context.docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	529 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	530 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	531 return true;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	532 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	533 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	534
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	535 private synchronized IndexSearcher openSearcher() throws IOException {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	536 int gwc = writeCounter.get();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	537 if( writeCount != gwc ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	538 writeCount = gwc;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	539 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	540 // DirectoryReader newReader = DirectoryReader.openIfChanged(reader,writer.getLuceneIndexWriter(),true);
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	541 if( newReader != null ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	542 reader.decRef();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	543 reader = newReader;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	544 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	545 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	546 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	547 reader.incRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	548 return searcher;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	549 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	550
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	551 // call in finally block
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	552 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	553 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	554 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	555
591 790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	556 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	557 close(openSearcher());
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	558 }
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	559
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	560 public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	561 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	562 {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	563 Utils.checkNotNull(queryStr);
1459 b04b8fc5f4f4 GoodQueryParser Franklin Schmidt <fschmidt@gmail.com> parents: 1458 diff changeset	564 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	565 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	566 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	567 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	568 searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	569 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	570 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	571 if( sortStr != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	572 throw new LuanException("sort must be nil when n is nil");
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	573 final DocFn docFn = new DocFn(searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	574 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	575 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	576 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	577 docFn.docID = docBase + doc;
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	578 fn.call(luan,++i,docFn);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	579 } catch(LuanException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	580 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	581 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	582 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	583 };
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	584 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	585 searcher.search(query,col);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	586 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	587 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	588 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	589 return col.i;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	590 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	591 if( fn==null \|\| n==0 ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	592 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	593 searcher.search(query,thcc);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	594 return thcc.getTotalHits();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	595 }
1459 b04b8fc5f4f4 GoodQueryParser Franklin Schmidt <fschmidt@gmail.com> parents: 1458 diff changeset	596 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	597 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	598 final ScoreDoc[] scoreDocs = td.scoreDocs;
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	599 DocFn docFn = new DocFn(searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	600 for( int i=0; i<scoreDocs.length; i++ ) {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	601 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	602 docFn.docID = scoreDoc.doc;
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	603 fn.call(luan,i+1,docFn,scoreDoc.score);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	604 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	605 return td.totalHits;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	606 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	607 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	608 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	609 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	610 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	611
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	612 public Object search_in_transaction(Luan luan,LuanFunction fn) throws LuanException, IOException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	613 if( threadLocalSearcher.get() != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	614 throw new LuanException("can't nest search_in_transaction calls");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	615 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	616 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	617 try {
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	618 return fn.call(luan);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	619 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	620 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	621 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	622 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	623 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	624
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	625
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	626 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	627 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	628 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	629
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	630 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	631 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	632 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	633 return;
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	634 }
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	635 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	636 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	637
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	638 static Map<String,Object> toLucene(LuanTable table) throws LuanException {
1529 e6d808f40bbc minor Franklin Schmidt <fschmidt@gmail.com> parents: 1528 diff changeset	639 return SupplementingConfig.toLucene(table);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	640 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	641
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	642 private static LuanTable toTable(Document doc) throws LuanException {
b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	643 return doc==null ? null : SupplementingConfig.toTable(LuceneUtils.toMap(doc));
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	644 }
d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	645
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	646
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	647 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	648 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	649 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	650 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	651 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	652
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	653 public LuanFunction highlighter(final Luan luan,String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	654 throws ParseException
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	655 {
1459 b04b8fc5f4f4 GoodQueryParser Franklin Schmidt <fschmidt@gmail.com> parents: 1458 diff changeset	656 Query query = GoodQueryParser.parseQuery(mfp,queryStr);
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	657 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	658 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1435 3aae37efbf40 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1430 diff changeset	659 if( tokenGroup.getTotalScore() <= 0 )
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	660 return originalText;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	661 try {
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	662 return (String)Luan.first(formatter.call(luan,originalText));
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	663 } catch(LuanException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	664 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	665 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	666 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	667 };
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	668 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	669 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	670 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	671 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	672 final Highlighter hl = new Highlighter(fmt,queryScorer);
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	673 hl.setTextFragmenter( new NullFragmenter() );
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	674 return new LuanFunction() {
8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	675 @Override public String call(Luan luan,Object[] args) throws LuanException {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	676 String text = (String)args[0];
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	677 try {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	678 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	679 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	680 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	681 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	682 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	683 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	684 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	685 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	686 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	687 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	688 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	689 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	690 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	691 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	692 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	693 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	694 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	695 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	696 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	697 return s!=null ? s : text;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	698 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	699 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	700 } catch(IOException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	701 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	702 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	703 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	704 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	705 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	706 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	707 }
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	708
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	709 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	710 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	711 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	712 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	713 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	714 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	715 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	716 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	717 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	718 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	719 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	720 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	721
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	722
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	723
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	724 public boolean hasPostgresBackup() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	725 return postgresBackup != null;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	726 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	727
1453 928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	728 public void rebuild_postgres_backup(Luan luan)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	729 throws IOException, LuanException, SQLException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	730 {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	731 logger.info("start rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	732 writeLock.lock();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	733 IndexSearcher searcher = openSearcher();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	734 boolean ok = false;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	735 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	736 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	737 postgresBackup.deleteAll();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	738 Query query = new PrefixQuery(new Term("id"));
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	739 MyCollector col = new MyCollector() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	740 @Override public void collect(int iDoc) throws IOException {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	741 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	742 Document doc = searcher.doc( docBase + iDoc );
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	743 LuanTable tbl = toTable(doc);
b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	744 postgresBackup.add(luan,tbl);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	745 } catch(LuanException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	746 throw new LuanRuntimeException(e);
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	747 } catch(SQLException e) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	748 throw new RuntimeException(e);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	749 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	750 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	751 };
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	752 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	753 searcher.search(query,col);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	754 } catch(LuanRuntimeException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	755 throw (LuanException)e.getCause();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	756 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	757 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	758 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	759 } finally {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	760 close(searcher);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	761 if( !ok )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	762 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	763 writeLock.unlock();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	764 }
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	765 logger.info("end rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	766 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	767
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	768 public void restore_from_postgres()
1447 851b9a48cc44 Luan.parse Franklin Schmidt <fschmidt@gmail.com> parents: 1435 diff changeset	769 throws IOException, LuanException, SQLException, ParseException
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	770 {
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	771 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	772 logger.error("restoring from postgres");
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	773 force_restore_from_postgres();
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	774 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	775 }
0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	776
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	777 public void force_restore_from_postgres()
1447 851b9a48cc44 Luan.parse Franklin Schmidt <fschmidt@gmail.com> parents: 1435 diff changeset	778 throws IOException, LuanException, SQLException, ParseException
1397 0dc9837c16be fix lucene issues Franklin Schmidt <fschmidt@gmail.com> parents: 1395 diff changeset	779 {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	780 logger.warn("start restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	781 if( postgresBackup==null )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	782 throw new NullPointerException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	783 if( writeLock.isHeldByCurrentThread() )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	784 throw new RuntimeException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	785 writeLock.lock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	786 boolean ok = false;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	787 try {
1548 736ec76bbf42 lucene log work Franklin Schmidt <fschmidt@gmail.com> parents: 1544 diff changeset	788 writer.tag("restore_from_postgres");
736ec76bbf42 lucene log work Franklin Schmidt <fschmidt@gmail.com> parents: 1544 diff changeset	789 writer.deleteAll();
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	790 postgresBackup.restoreLucene(this);
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	791 ok = true;
1548 736ec76bbf42 lucene log work Franklin Schmidt <fschmidt@gmail.com> parents: 1544 diff changeset	792 writer.commit();
1539 c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	793 wrote();
c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	794 ensure_open(); // refresh searcher
c27dc6af87ca remove nextId Franklin Schmidt <fschmidt@gmail.com> parents: 1538 diff changeset	795 initId();
1398 67c0e47b5be3 more lucene Franklin Schmidt <fschmidt@gmail.com> parents: 1397 diff changeset	796 wasCreated = false;
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	797 } finally {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	798 if( !ok ) {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	799 writer.rollback();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	800 reopen();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	801 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	802 wrote();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	803 writeLock.unlock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	804 }
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	805 logger.warn("end restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	806 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	807
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	808 void restore(LuanTable doc)
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	809 throws LuanException, IOException
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	810 {
1528 3bd4d7963456 use goodjava/lucene/api Franklin Schmidt <fschmidt@gmail.com> parents: 1526 diff changeset	811 writer.addDocument(toLucene(doc));
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	812 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	813
1556 52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	814 public void relog()
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	815 throws IOException, LuanException
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	816 {
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	817 logger.info("start relog");
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	818 writeLock.lock();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	819 try {
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	820 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	821 loggingWriter.logLucene();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	822 } finally {
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	823 writeLock.unlock();
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	824 }
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	825 logger.info("end relog");
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	826 }
52241b69c339 lucene logging Franklin Schmidt <fschmidt@gmail.com> parents: 1553 diff changeset	827
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	828 public void restore_from_log(Luan luan,LuanFunction handler)
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	829 throws IOException, LuanException, SQLException, ParseException
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	830 {
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	831 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	832 if( wasCreated && !loggingWriter.wasCreated ) {
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	833 logger.error("restoring from log");
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	834 force_restore_from_log(luan,handler);
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	835 }
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	836 }
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	837
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	838 public void force_restore_from_log(Luan luan,LuanFunction handler)
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	839 throws IOException
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	840 {
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	841 logger.warn("start force_restore_from_log");
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	842 if( writeLock.isHeldByCurrentThread() )
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	843 throw new RuntimeException();
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	844 OpDoer opDoer = handler==null ? null : new LuanOpDoer(writer,luan,handler);
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	845 writeLock.lock();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	846 boolean ok = false;
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	847 try {
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	848 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
1551 9cc4cee39b8b add LuanOpDoer Franklin Schmidt <fschmidt@gmail.com> parents: 1548 diff changeset	849 loggingWriter.playLogs(opDoer);
1544 35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	850 ok = true;
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	851 wrote();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	852 ensure_open(); // refresh searcher
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	853 initId();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	854 wasCreated = false;
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	855 } finally {
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	856 if( !ok ) {
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	857 writer.rollback();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	858 reopen();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	859 }
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	860 wrote();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	861 writeLock.unlock();
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	862 }
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	863 logger.warn("end force_restore_from_log");
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	864 }
35601f15ecc3 add lucene log tag and restore_from_log Franklin Schmidt <fschmidt@gmail.com> parents: 1543 diff changeset	865
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	866 public void check() throws IOException, SQLException, LuanException, ParseException {
1406 8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	867 boolean hasPostgres = postgresBackup != null;
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	868 String msg = "start check";
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	869 if( hasPostgres )
8187ddb0e827 minor Franklin Schmidt <fschmidt@gmail.com> parents: 1402 diff changeset	870 msg += " with postgres";
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	871 logger.info(msg);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	872 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	873 if( !status.clean )
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	874 logger.error("index not clean");
1538 634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	875 if( writer instanceof LoggingIndexWriter ) {
634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	876 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	877 logger.info("log check");
634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	878 boolean ok = loggingWriter.check(ID_SORT);
634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	879 }
634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	880 if( hasPostgres ) {
634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	881 logger.info("postgres check");
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	882 checkPostgres();
1538 634f6765830e use goodjava/lucene/logging Franklin Schmidt <fschmidt@gmail.com> parents: 1535 diff changeset	883 }
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	884 logger.info("end check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	885 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	886
1563 8fbcc4747091 remove LuanFunction.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1562 diff changeset	887 private void checkPostgres()
1447 851b9a48cc44 Luan.parse Franklin Schmidt <fschmidt@gmail.com> parents: 1435 diff changeset	888 throws IOException, SQLException, LuanException, ParseException
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	889 {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	890 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	891 final IndexSearcher searcher = openSearcher();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	892 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	893 final List<Long> idsLucene = new ArrayList<Long>();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	894 Query query = new PrefixQuery(new Term("id"));
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	895 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	896 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	897 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	898 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	899 idsLucene.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	900 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	901 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	902 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	903 Collections.sort(idsLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	904 final List<Long> idsPostgres = postgresChecker.getIds();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	905 final int nLucene = idsLucene.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	906 final int nPostgres = idsPostgres.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	907 int iLucene = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	908 int iPostgres = 0;
1623 582384548a69 LuanToString takes function Franklin Schmidt <fschmidt@gmail.com> parents: 1563 diff changeset	909 LuanToString lts = new LuanToString();
1553 26c51acf00f3 improve stringify Franklin Schmidt <fschmidt@gmail.com> parents: 1551 diff changeset	910 lts.settingsInit.strict = true;
26c51acf00f3 improve stringify Franklin Schmidt <fschmidt@gmail.com> parents: 1551 diff changeset	911 lts.settingsInit.numberTypes = true;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	912 while( iLucene < nLucene && iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	913 long idLucene = idsLucene.get(iLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	914 long idPostgres = idsPostgres.get(iPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	915 if( idLucene < idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	916 iLucene++;
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	917 checkPostgres(postgresChecker,lts,idLucene);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	918 } else if( idLucene > idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	919 iPostgres++;
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	920 checkPostgres(postgresChecker,lts,idPostgres);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	921 } else { // ==
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	922 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	923 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	924 if( td.totalHits != 1 ) throw new RuntimeException();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	925 Document doc = searcher.doc( td.scoreDocs[0].doc );
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	926 LuanTable docLucene = toTable(doc);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	927 if( !equal(docPostgres,docLucene) ) {
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	928 checkPostgres(postgresChecker,lts,idPostgres);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	929 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	930 iLucene++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	931 iPostgres++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	932 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	933 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	934 while( iLucene < nLucene ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	935 long idLucene = idsLucene.get(iLucene++);
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	936 checkPostgres(postgresChecker,lts,idLucene);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	937 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	938 while( iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	939 long idPostgres = idsPostgres.get(iPostgres++);
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	940 checkPostgres(postgresChecker,lts,idPostgres);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	941 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	942 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	943 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	944 postgresChecker.close();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	945 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	946 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	947
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	948 private void checkPostgres(PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
1447 851b9a48cc44 Luan.parse Franklin Schmidt <fschmidt@gmail.com> parents: 1435 diff changeset	949 throws IOException, SQLException, LuanException, ParseException
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	950 {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	951 //logger.info("check id "+id);
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	952 writeLock.lock();
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	953 try {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	954 final IndexSearcher searcher = openSearcher();
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	955 try {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	956 LuanTable docPostgres = postgresChecker.getDoc(id);
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	957 TopDocs td = searcher.search(new TermQuery(term("id",id)),1);
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	958 LuanTable docLucene;
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	959 if( td.totalHits == 0 ) {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	960 docLucene = null;
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	961 } else if( td.totalHits == 1 ) {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	962 Document doc = searcher.doc( td.scoreDocs[0].doc );
1562 b89212fd04b5 remove table.luan Franklin Schmidt <fschmidt@gmail.com> parents: 1556 diff changeset	963 docLucene = toTable(doc);
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	964 } else
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	965 throw new RuntimeException();
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	966 if( docPostgres == null ) {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	967 if( docLucene != null )
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	968 logger.error("id "+id+" found in lucene but not postgres");
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	969 return;
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	970 }
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	971 if( docLucene == null ) {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	972 logger.error("id "+id+" found in postgres but not lucene");
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	973 return;
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	974 }
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	975 if( !equal(docPostgres,docLucene) ) {
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	976 logger.error("id "+id+" not equal");
219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	977 logger.error("lucene = "+lts.toString(docLucene));
219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	978 logger.error("postgres = "+lts.toString(docPostgres));
1430 103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	979 }
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	980 } finally {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	981 close(searcher);
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	982 }
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	983 } finally {
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	984 writeLock.unlock();
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	985 }
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	986 }
103d0ce70385 fix postgres check Franklin Schmidt <fschmidt@gmail.com> parents: 1420 diff changeset	987
1453 928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	988 private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	989 return t1!=null && t2!=null && toJava(t1).equals(toJava(t2));
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	990 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	991
1453 928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	992 private static Map toJava(LuanTable t) throws LuanException {
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	993 Map map = t.asMap();
1526 efbc3720d3f3 postgres backup fix Franklin Schmidt <fschmidt@gmail.com> parents: 1482 diff changeset	994 for( Iterator iter = map.entrySet().iterator(); iter.hasNext(); ) {
efbc3720d3f3 postgres backup fix Franklin Schmidt <fschmidt@gmail.com> parents: 1482 diff changeset	995 Map.Entry entry = (Map.Entry)iter.next();
1453 928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	996 Object value = entry.getValue();
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	997 if( value instanceof LuanTable ) {
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	998 LuanTable v = (LuanTable)value;
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	999 if( !v.isList() )
1454 219f2b937f2b remove log4j Franklin Schmidt <fschmidt@gmail.com> parents: 1453 diff changeset	1000 logger.error("not list");
1530 447b7ef9197f minor fix Franklin Schmidt <fschmidt@gmail.com> parents: 1529 diff changeset	1001 List list = v.asList();
447b7ef9197f minor fix Franklin Schmidt <fschmidt@gmail.com> parents: 1529 diff changeset	1002 if( list.isEmpty() ) {
1526 efbc3720d3f3 postgres backup fix Franklin Schmidt <fschmidt@gmail.com> parents: 1482 diff changeset	1003 iter.remove();
1530 447b7ef9197f minor fix Franklin Schmidt <fschmidt@gmail.com> parents: 1529 diff changeset	1004 } else if( list.size() == 1 ) {
447b7ef9197f minor fix Franklin Schmidt <fschmidt@gmail.com> parents: 1529 diff changeset	1005 entry.setValue(list.get(0));
1526 efbc3720d3f3 postgres backup fix Franklin Schmidt <fschmidt@gmail.com> parents: 1482 diff changeset	1006 } else {
1530 447b7ef9197f minor fix Franklin Schmidt <fschmidt@gmail.com> parents: 1529 diff changeset	1007 entry.setValue(list);
1526 efbc3720d3f3 postgres backup fix Franklin Schmidt <fschmidt@gmail.com> parents: 1482 diff changeset	1008 }
1453 928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	1009 }
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	1010 }
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	1011 return map;
928be2a4d565 fix postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1447 diff changeset	1012 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1013 }

Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1687:f48db13ae2d9