repo/luan: src/luan/modules/lucene/LuceneIndex.java annotate

annotate src/luan/modules/lucene/LuceneIndex.java @ 1393:cc0dbca576dc

better logging

author	Franklin Schmidt <fschmidt@gmail.com>
date	Fri, 06 Sep 2019 05:09:56 -0600
parents	002152af497a
children	9dfff82dfc59

rev	line source
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	1 package luan.modules.lucene;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	2
527 ef0336efe33c minor Franklin Schmidt <fschmidt@gmail.com> parents: 524 diff changeset	3 import java.io.Closeable;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	4 import java.io.File;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	6 import java.io.FileInputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	7 import java.io.IOException;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	8 import java.sql.SQLException;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	9 import java.util.Arrays;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	10 import java.util.Iterator;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	11 import java.util.Map;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	12 import java.util.HashMap;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	13 import java.util.List;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	14 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	15 import java.util.Set;
618 5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	16 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields Franklin Schmidt <fschmidt@gmail.com> parents: 617 diff changeset	17 import java.util.Collections;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	18 import java.util.concurrent.atomic.AtomicInteger;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	19 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	20 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	21 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	22 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	23 import org.apache.lucene.analysis.Analyzer;
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	24 import org.apache.lucene.analysis.TokenStream;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	25 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	26 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	27 import org.apache.lucene.document.Document;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	28 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	29 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	30 import org.apache.lucene.document.StringField;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	31 import org.apache.lucene.document.TextField;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	32 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	33 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	34 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	35 import org.apache.lucene.index.IndexableField;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	36 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	37 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	38 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	39 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	40 import org.apache.lucene.index.SnapshotDeletionPolicy;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	41 import org.apache.lucene.index.IndexCommit;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	42 import org.apache.lucene.index.AtomicReaderContext;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	43 import org.apache.lucene.index.CheckIndex;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	44 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	45 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	46 import org.apache.lucene.util.Version;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	47 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	48 import org.apache.lucene.util.NumericUtils;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	49 import org.apache.lucene.search.Query;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	50 import org.apache.lucene.search.PrefixQuery;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	51 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	52 import org.apache.lucene.search.TopDocs;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	53 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	54 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	55 import org.apache.lucene.search.IndexSearcher;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	56 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	57 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	58 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	59 import org.apache.lucene.search.Scorer;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	60 import org.apache.lucene.search.Explanation;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	61 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	62 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	63 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	64 import org.apache.lucene.search.highlight.Fragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	65 import org.apache.lucene.search.highlight.NullFragmenter;
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	66 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	67 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	68 import org.apache.lucene.search.highlight.TokenGroup;
1344 dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	69 import luan.lib.queryparser.SaneQueryParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	70 import luan.lib.queryparser.FieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	71 import luan.lib.queryparser.MultiFieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	72 import luan.lib.queryparser.StringFieldParser;
dc2af9d5463b move queryparser to lib Franklin Schmidt <fschmidt@gmail.com> parents: 1343 diff changeset	73 import luan.lib.queryparser.NumberFieldParser;
1111 88b5b81cad4a move Parser to luan.lib.parser Franklin Schmidt <fschmidt@gmail.com> parents: 1110 diff changeset	74 import luan.lib.parser.ParseException;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	75 import luan.modules.Utils;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	76 import luan.Luan;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	77 import luan.LuanTable;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	78 import luan.LuanFunction;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	79 import luan.LuanException;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	80 import luan.LuanRuntimeException;
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	81 import luan.modules.parsers.LuanToString;
1337 8b61c8c4e07a remove slf4j Franklin Schmidt <fschmidt@gmail.com> parents: 1335 diff changeset	82 import luan.lib.logging.Logger;
8b61c8c4e07a remove slf4j Franklin Schmidt <fschmidt@gmail.com> parents: 1335 diff changeset	83 import luan.lib.logging.LoggerFactory;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	84
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	85
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	86 public final class LuceneIndex {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	87 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	88
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	89 private static final class Closer implements Closeable {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	90 final LuceneIndex li;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	91 boolean isClosed = false;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	92 private final Exception created = new Exception("created");
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	93
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	94 Closer(Luan luan,LuceneIndex li) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	95 this.li = li;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	96 luan.onClose(this);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	97 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	98
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	99 public void close() throws IOException {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	100 if( !isClosed ) {
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	101 try {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	102 li.close();
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	103 } catch(SQLException e) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	104 throw new RuntimeException(e);
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	105 }
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	106 isClosed = true;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	107 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	108 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	109
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	110 protected void finalize() throws Throwable {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	111 if( !isClosed ) {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	112 sysLogger.error("not closed",created);
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	113 close();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	114 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	115 super.finalize();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	116 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	117 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	118
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	119 private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	120
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	121 public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,LuanTable postgresSpec)
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	122 throws LuanException, IOException, ClassNotFoundException, SQLException
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	123 {
1369 709f7498a363 change Lucene.index() and add Lucene.recover() Franklin Schmidt <fschmidt@gmail.com> parents: 1347 diff changeset	124 String key = indexDir.getCanonicalPath();
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	125 synchronized(indexes) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	126 LuceneIndex li = indexes.get(key);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	127 if( li == null ) {
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	128 li = new LuceneIndex(luan,indexDir,defaultFieldParser,defaultFields,key,postgresSpec);
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	129 li.openCount = 1;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	130 indexes.put(key,li);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	131 } else {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	132 if( defaultFieldParser != li.defaultFieldParser )
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	133 throw new LuanException("default_type doesn't match previous use");
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	134 if( !Arrays.equals(defaultFields,li.defaultFields) )
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	135 throw new LuanException("default_fields don't match previous use");
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	136 li.openCount++;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	137 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	138 return new Object[]{li,new Closer(luan,li)};
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	139 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	140 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	141
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	142 private static final Version version = Version.LUCENE_4_9;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	143 private static final String FLD_NEXT_ID = "nextId";
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	144 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	145 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	146
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	147 private final Logger luanLogger;
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	148
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	149 private final ReentrantLock writeLock = new ReentrantLock();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	150 private final File indexDir;
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	151 private SnapshotDeletionPolicy snapshotDeletionPolicy;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	152 private IndexWriter writer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	153 private DirectoryReader reader;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	154 private IndexSearcher searcher;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	155 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	156 private final MultiFieldParser mfp;
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	157 private final Analyzer analyzer;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	158
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	159 private FSDirectory fsDir;
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	160 private int writeCount;
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	161 private AtomicInteger writeCounter = new AtomicInteger();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	162
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	163 private Set<String> indexOnly = new HashSet<String>();
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	164
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	165 private int openCount;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	166 private final String key;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	167 private final FieldParser defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	168 private final String[] defaultFields;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	169
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	170 private final PostgresBackup postgresBackup;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	171
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	172 private LuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanTable postgresSpec)
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	173 throws LuanException, IOException, ClassNotFoundException, SQLException
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	174 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	175 this.luanLogger = luan.getLogger(LuceneIndex.class);
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	176 this.key = key;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	177 this.defaultFieldParser = defaultFieldParser;
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	178 this.defaultFields = defaultFields;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	179 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	180 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	181 mfp.fields.put( "id", NumberFieldParser.LONG );
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	182 this.indexDir = indexDir;
599 50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	183 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	184 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	185 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	186 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene; Franklin Schmidt <fschmidt@gmail.com> parents: 591 diff changeset	187 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	188 this.analyzer = analyzer;
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	189 boolean wasCreated = reopen();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	190 if( postgresSpec == null ) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	191 postgresBackup = null;
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	192 } else {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	193 Map spec = postgresSpec.asMap();
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	194 LuanFunction completer = Utils.removeRequiredFunction(spec,"completer");
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	195 postgresBackup = new PostgresBackup(luan,spec);
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	196 if( postgresBackup != null ) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	197 if( !wasCreated && postgresBackup.wasCreated ) {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	198 luanLogger.error("rebuilding postgres backup");
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	199 rebuild_postgres_backup(completer);
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	200 } else if( wasCreated && !postgresBackup.wasCreated ) {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	201 luanLogger.error("restoring from postgres");
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	202 restore_from_postgres();
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	203 }
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	204 }
179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	205 }
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	206 }
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	207
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	208 public boolean reopen() throws IOException {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	209 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	210 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	211 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	212 fsDir = FSDirectory.open(indexDir);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	213 boolean wasCreated = !fsDir.getDirectory().exists();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	214 writer = new IndexWriter(fsDir,conf);
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	215 writer.commit(); // commit index creation
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	216 reader = DirectoryReader.open(fsDir);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	217 searcher = new IndexSearcher(reader);
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	218 initId();
1390 179c4882c6b6 backup work Franklin Schmidt <fschmidt@gmail.com> parents: 1388 diff changeset	219 return wasCreated;
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	220 }
ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	221
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	222 private void wrote() {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	223 writeCounter.incrementAndGet();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	224 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	225
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	226 public void delete_all() throws IOException, SQLException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	227 boolean commit = !writeLock.isHeldByCurrentThread();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	228 writeLock.lock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	229 try {
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	230 writer.deleteAll();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	231 id = idLim = 0;
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	232 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	233 postgresBackup.deleteAll();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	234 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	235 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	236 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	237 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	238 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	239 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	240
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	241 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	242 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	243 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	244 return new Term(key,br);
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	245 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	246
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	247 private void backupDelete(Query query)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	248 throws IOException, SQLException, LuanException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	249 {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	250 if( postgresBackup != null ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	251 final List<Long> ids = new ArrayList<Long>();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	252 IndexSearcher searcher = openSearcher();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	253 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	254 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	255 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	256 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	257 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	258 ids.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	259 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	260 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	261 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	262 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	263 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	264 }
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	265 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	266 for( Long id : ids ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	267 postgresBackup.delete(id);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	268 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	269 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	270 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	271 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	272
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	273 public void delete(String queryStr)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	274 throws IOException, ParseException, SQLException, LuanException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	275 {
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	276 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	277
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	278 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	279 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	280 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	281 backupDelete(query);
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	282 writer.deleteDocuments(query);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	283 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	284 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	285 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	286 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	287 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	288 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	289
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	290 public void indexed_only_fields(List<String> fields) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	291 indexOnly.addAll(fields);
1271 48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	292 }
48f302bdc187 fix indexed_only_field Franklin Schmidt <fschmidt@gmail.com> parents: 1267 diff changeset	293
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	294 public void save(LuanTable doc,LuanTable boosts)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	295 throws LuanException, IOException, SQLException
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	296 {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	297 if( boosts!=null && postgresBackup!=null )
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	298 throw new LuanException("boosts are not saved to postgres backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	299
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	300 Object obj = doc.get("id");
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	301 Long id;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	302 try {
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	303 id = (Long)obj;
d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	304 } catch(ClassCastException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	305 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601 d36027b41570 better error message Franklin Schmidt <fschmidt@gmail.com> parents: 599 diff changeset	306 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	307
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	308 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	309 writeLock.lock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	310 try {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	311 if( id == null ) {
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	312 id = nextId();
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	313 doc.put("id",id);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	314 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	315 postgresBackup.add(doc);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	316 writer.addDocument(toLucene(doc,boosts));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	317 } else {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	318 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	319 postgresBackup.update(doc);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	320 writer.updateDocument( term("id",id), toLucene(doc,boosts) );
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	321 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	322 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	323 } finally {
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	324 wrote();
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	325 writeLock.unlock();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	326 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	327 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	328
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	329 public Object run_in_transaction(LuanFunction fn)
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	330 throws IOException, LuanException, SQLException
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	331 {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	332 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	333 writeLock.lock();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	334 boolean ok = false;
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	335 try {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	336 if( commit && postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	337 postgresBackup.begin();
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	338 Object rtn = fn.call();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	339 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	340 if(commit) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	341 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	342 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	343 writer.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	344 }
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	345 return rtn;
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	346 } finally {
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	347 if( !ok && commit ) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	348 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	349 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	350 writer.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	351 reopen();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	352 }
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	353 wrote();
252 3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	354 writeLock.unlock();
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	355 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	356 }
3896138955b1 web testing... fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 233 diff changeset	357
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	358 // ???
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	359 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException {
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	360 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	361 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	362 writeLock.lock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	363 try {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	364 synchronized(this) {
1379 87a3738d7cc5 run_in_transaction Franklin Schmidt <fschmidt@gmail.com> parents: 1369 diff changeset	365 return fn.call();
756 9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	366 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	367 } finally {
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	368 wrote();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	369 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	370 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	371 }
9092e52f94eb better synchronization for lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 754 diff changeset	372
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	373
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	374 private long id;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	375 private long idLim;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	376 private final int idBatch = 10;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	377
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	378 private void initId() throws IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	379 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	380 switch(td.totalHits) {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	381 case 0:
754 1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	382 id = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	383 idLim = 0;
1a101ac9ea46 add lucene restore Franklin Schmidt <fschmidt@gmail.com> parents: 730 diff changeset	384 break;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	385 case 1:
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	386 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	387 id = idLim;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	388 break;
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	389 default:
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	390 throw new RuntimeException();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	391 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	392 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	393
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	394 private void saveNextId(long nextId) throws LuanException, IOException {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	395 Map doc = new HashMap();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	396 doc.put( "type", "next_id" );
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	397 doc.put( FLD_NEXT_ID, idLim );
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	398 writer.updateDocument(new Term("type","next_id"),toLucene(doc.entrySet(),null));
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	399 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	400
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	401 public synchronized long nextId() throws LuanException, IOException {
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	402 if( ++id > idLim ) {
233 ef39bc4d3f70 basic lucene works fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 232 diff changeset	403 idLim += idBatch;
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	404 saveNextId(idLim);
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	405 wrote();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	406 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	407 return id;
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	408 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	409
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	410 /*
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	411 public void backup(String zipFile) throws LuanException, IOException {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	412 if( !zipFile.endsWith(".zip") )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	413 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	414 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	415 try {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	416 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	417 for( String fileName : ic.getFileNames() ) {
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	418 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	419 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	420 Utils.copyAll(in,out);
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	421 in.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	422 out.closeEntry();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	423 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	424 out.close();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	425 } finally {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	426 snapshotDeletionPolicy.release(ic);
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	427 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	428 }
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	429 */
1129 3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	430 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	431 return snapshotDeletionPolicy;
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	432 }
3234a14bb1f8 minor lucene changes Franklin Schmidt <fschmidt@gmail.com> parents: 1111 diff changeset	433
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	434 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	435 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	436 try {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	437 String dir = fsDir.getDirectory().toString();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	438 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames()));
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	439 return fn.call(dir,fileNames);
707 1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	440 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	441 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	442 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot Franklin Schmidt <fschmidt@gmail.com> parents: 704 diff changeset	443 }
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	444
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	445
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	446
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	447 public String to_string() {
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	448 return writer.getDirectory().toString();
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	449 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	450
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	451 private synchronized void close() throws IOException, SQLException {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	452 if( openCount > 0 ) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	453 if( --openCount == 0 ) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	454 doClose();
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	455 synchronized(indexes) {
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	456 indexes.remove(key);
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	457 }
6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	458 }
521 8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	459 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	460 }
8a217fe5b4f3 cleaner LuanState.onClose() Franklin Schmidt <fschmidt@gmail.com> parents: 427 diff changeset	461
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	462 public void doClose() throws IOException, SQLException {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	463 writer.close();
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	464 reader.close();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	465 if( postgresBackup != null )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	466 postgresBackup.close();
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	467 }
4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	468
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	469
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	470 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	471 final IndexSearcher searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	472 final Query query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	473 int docID;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	474
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	475 DocFn(Luan luan,IndexSearcher searcher,Query query) {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	476 super(luan);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	477 this.searcher = searcher;
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	478 this.query = query;
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	479 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	480
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	481 @Override public Object call(Object[] args) throws LuanException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	482 try {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	483 LuanTable doc = toTable(luan(),searcher.doc(docID));
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	484 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	485 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	486 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	487 } else {
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	488 return doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	489 }
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	490 } catch(IOException e) {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	491 throw new LuanException(e);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	492 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	493 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	494 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	495
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	496 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	497 int docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	498 int i = 0;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	499
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	500 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	501 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	502 this.docBase = context.docBase;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	503 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	504 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	505 return true;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	506 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	507 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	508
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	509 private synchronized IndexSearcher openSearcher() throws IOException {
1345 6f8988830098 unique LuceneIndex per dir Franklin Schmidt <fschmidt@gmail.com> parents: 1344 diff changeset	510 int gwc = writeCounter.get();
704 90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	511 if( writeCount != gwc ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	512 writeCount = gwc;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	513 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	514 if( newReader != null ) {
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	515 reader.decRef();
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	516 reader = newReader;
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	517 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene Franklin Schmidt <fschmidt@gmail.com> parents: 696 diff changeset	518 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	519 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	520 reader.incRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	521 return searcher;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	522 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	523
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	524 // call in finally block
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	525 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	526 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	527 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	528
591 790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	529 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	530 close(openSearcher());
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	531 }
790d5de23042 add "strict" param to Io.repr(); Franklin Schmidt <fschmidt@gmail.com> parents: 578 diff changeset	532
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	533 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	534 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	535 {
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	536 Utils.checkNotNull(queryStr);
622 1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this Franklin Schmidt <fschmidt@gmail.com> parents: 621 diff changeset	537 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	538 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	539 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	540 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	541 searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	542 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	543 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	544 if( sortStr != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	545 throw new LuanException("sort must be nil when n is nil");
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	546 final DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	547 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	548 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	549 try {
547 0be287ab0309 add lucene/Versioning and simplify Lucene fn names Franklin Schmidt <fschmidt@gmail.com> parents: 546 diff changeset	550 docFn.docID = docBase + doc;
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	551 fn.call(++i,docFn);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	552 } catch(LuanException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	553 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	554 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	555 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	556 };
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	557 try {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	558 searcher.search(query,col);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	559 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	560 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	561 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	562 return col.i;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	563 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	564 if( fn==null \|\| n==0 ) {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	565 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	566 searcher.search(query,thcc);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	567 return thcc.getTotalHits();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	568 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	569 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	570 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	571 final ScoreDoc[] scoreDocs = td.scoreDocs;
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	572 DocFn docFn = new DocFn(fn.luan(),searcher,query);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	573 for( int i=0; i<scoreDocs.length; i++ ) {
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	574 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	575 docFn.docID = scoreDoc.doc;
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	576 fn.call(i+1,docFn,scoreDoc.score);
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	577 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	578 return td.totalHits;
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	579 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	580 if( !inTransaction )
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	581 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	582 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	583 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	584
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	585 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException {
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	586 if( threadLocalSearcher.get() != null )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	587 throw new LuanException("can't nest search_in_transaction calls");
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	588 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	589 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	590 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	591 return fn.call();
545 ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	592 } finally {
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	593 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	594 close(searcher);
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	595 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	596 }
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	597
ddcd4296107a clean up lucene search Franklin Schmidt <fschmidt@gmail.com> parents: 544 diff changeset	598
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	599 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	600 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	601 }
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	602
796 6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	603 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	604 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	605 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	606 return;
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	607 }
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	608 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta Franklin Schmidt <fschmidt@gmail.com> parents: 775 diff changeset	609 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	610
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	611
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	612 private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	613 throws LuanException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	614 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	615 boolean hasBoost = boost!=null;
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	616 IndexableField fld = newField2(name,value,indexed,hasBoost);
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	617 if( hasBoost )
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	618 ((Field)fld).setBoost(boost);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	619 return fld;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	620 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	621
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	622 private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	623 throws LuanException
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	624 {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	625 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	626 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	627 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	628 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	629 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	630 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	631 return new TextField(name, s, store);
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	632 } else if (hasBoost) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	633 // fuck you modern lucene developers
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	634 return new Field(name, s, store, Field.Index.NOT_ANALYZED);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	635 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	636 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	637 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	638 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	639 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	640 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	641 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	642 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	643 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	644 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	645 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	646 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	647 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	648 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	649 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	650 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	651 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	652 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	653 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	654 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	655 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	656 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	657 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	658 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	659 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	660 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	661 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	662 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	663 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	664 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	665 } else
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	666 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	667 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	668
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	669 private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
1347 643cf1c37723 move webserver to lib and bug fixes Franklin Schmidt <fschmidt@gmail.com> parents: 1346 diff changeset	670 return toLucene(table.iterable(),boosts);
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	671 }
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	672
efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	673 private Document toLucene(Iterable<Map.Entry> iterable,LuanTable boosts) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	674 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	675 Document doc = new Document();
1346 efd1c6380f2c minor Franklin Schmidt <fschmidt@gmail.com> parents: 1345 diff changeset	676 for( Map.Entry<Object,Object> entry : iterable ) {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	677 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	678 if( !(key instanceof String) )
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	679 throw new LuanException("key must be string");
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	680 String name = (String)key;
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	681 Object value = entry.getValue();
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	682 Float boost = null;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	683 if( boosts != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	684 Object obj = boosts.get(name);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	685 if( obj != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	686 if( !(obj instanceof Number) )
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	687 throw new LuanException("boost '"+name+"' must be number");
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	688 boost = ((Number)obj).floatValue();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	689 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	690 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	691 if( !(value instanceof LuanTable) ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	692 doc.add(newField( name, value, indexed, boost ));
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	693 } else { // list
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	694 LuanTable list = (LuanTable)value;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	695 for( Object el : list.asList() ) {
1343 7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields() Franklin Schmidt <fschmidt@gmail.com> parents: 1342 diff changeset	696 doc.add(newField( name, el, indexed, boost ));
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	697 }
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	698 }
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	699 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	700 return doc;
544 c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	701 }
c5a93767cc5c lucene overhaul, untested Franklin Schmidt <fschmidt@gmail.com> parents: 527 diff changeset	702
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	703 private static Object getValue(IndexableField ifld) throws LuanException {
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	704 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	705 if( br != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	706 return br.bytes;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	707 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	708 if( n != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	709 return n;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	710 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	711 if( s != null )
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	712 return s;
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	713 throw new LuanException("invalid field type for "+ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	714 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	715
1333 25746915a241 merge Luan and LuanState Franklin Schmidt <fschmidt@gmail.com> parents: 1271 diff changeset	716 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	717 if( doc==null )
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	718 return null;
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	719 LuanTable table = new LuanTable(luan);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	720 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	721 String name = ifld.name();
646 cdc70de628b5 simplify LuanException Franklin Schmidt <fschmidt@gmail.com> parents: 624 diff changeset	722 Object value = getValue(ifld);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	723 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	724 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	725 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	726 } else {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	727 LuanTable list;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	728 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	729 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	730 } else {
1267 9fa8b8389578 add LuanTable.luan; Franklin Schmidt <fschmidt@gmail.com> parents: 1248 diff changeset	731 list = new LuanTable(luan);
621 fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	732 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	733 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	734 }
fd15da41afca allow list of values to be stored in lucene Franklin Schmidt <fschmidt@gmail.com> parents: 620 diff changeset	735 list.rawPut(list.rawLength()+1,value);
546 eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	736 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	737 }
eaef1005ab87 general lucene cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 545 diff changeset	738 return table;
312 d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	739 }
d34be4588556 add lucene query parsing fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: 303 diff changeset	740
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	741
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	742 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	743 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	744 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	745 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	746 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	747
1342 60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	748 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	749 throws ParseException
60599adc27b8 add lucene search options Franklin Schmidt <fschmidt@gmail.com> parents: 1341 diff changeset	750 {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	751 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	752 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	753 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	754 if( tokenGroup.getTotalScore() <= 0 )
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	755 return originalText;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	756 try {
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	757 return (String)Luan.first(formatter.call(originalText));
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	758 } catch(LuanException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	759 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	760 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	761 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	762 };
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	763 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	764 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	765 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	766 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	767 final Highlighter hl = new Highlighter(fmt,queryScorer);
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	768 hl.setTextFragmenter( new NullFragmenter() );
1335 e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	769 return new LuanFunction(false) { // ???
e0cf0d108a77 major cleanup Franklin Schmidt <fschmidt@gmail.com> parents: 1333 diff changeset	770 @Override public String call(Object[] args) throws LuanException {
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	771 String text = (String)args[0];
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	772 try {
1248 475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	773 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	774 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	775 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	776 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	777 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	778 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	779 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	780 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	781 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	782 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	783 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	784 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	785 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	786 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	787 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	788 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	789 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil Franklin Schmidt <fschmidt@gmail.com> parents: 1227 diff changeset	790 }
624 8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	791 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	792 return s!=null ? s : text;
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	793 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	794 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	795 } catch(IOException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	796 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	797 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	798 throw new RuntimeException(e);
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	799 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	800 }
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	801 };
8281a248c47e add lucene highlighter Franklin Schmidt <fschmidt@gmail.com> parents: 622 diff changeset	802 }
1341 a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	803
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	804 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	805 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	806 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	807 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	808 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	809 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	810 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	811 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	812 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	813 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	814 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	815 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set() Franklin Schmidt <fschmidt@gmail.com> parents: 1337 diff changeset	816
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	817
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	818
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	819 public boolean hasPostgresBackup() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	820 return postgresBackup != null;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	821 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	822
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	823 public void rebuild_postgres_backup(LuanFunction completer)
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	824 throws IOException, LuanException, SQLException
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	825 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	826 luanLogger.info("start rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	827 writeLock.lock();
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	828 IndexSearcher searcher = openSearcher();
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	829 boolean ok = false;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	830 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	831 postgresBackup.begin();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	832 postgresBackup.deleteAll();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	833 Query query = new PrefixQuery(new Term("id"));
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	834 MyCollector col = new MyCollector() {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	835 @Override public void collect(int iDoc) throws IOException {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	836 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	837 Document doc = searcher.doc( docBase + iDoc );
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	838 LuanTable tbl = toTable(completer.luan(),doc);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	839 tbl = (LuanTable)completer.call(tbl);
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	840 postgresBackup.add(tbl);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	841 } catch(LuanException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	842 throw new LuanRuntimeException(e);
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	843 } catch(SQLException e) {
002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	844 throw new RuntimeException(e);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	845 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	846 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	847 };
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	848 try {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	849 searcher.search(query,col);
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	850 } catch(LuanRuntimeException e) {
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	851 throw (LuanException)e.getCause();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	852 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	853 ok = true;
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	854 postgresBackup.commit();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	855 } finally {
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	856 close(searcher);
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	857 if( !ok )
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	858 postgresBackup.rollback();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	859 writeLock.unlock();
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	860 }
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	861 luanLogger.info("end rebuild_postgres_backup");
1387 bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	862 }
bc40bc9aab3a start postgres backup Franklin Schmidt <fschmidt@gmail.com> parents: 1379 diff changeset	863
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	864 public void restore_from_postgres()
1392 002152af497a hosted postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1391 diff changeset	865 throws IOException, LuanException, SQLException
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	866 {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	867 luanLogger.warn("start restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	868 if( postgresBackup==null )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	869 throw new NullPointerException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	870 if( writeLock.isHeldByCurrentThread() )
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	871 throw new RuntimeException();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	872 writeLock.lock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	873 boolean ok = false;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	874 try {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	875 writer.deleteAll();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	876 long nextId = postgresBackup.maxId() + 1;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	877 postgresBackup.restoreLucene(this);
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	878 id = idLim = nextId;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	879 ok = true;
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	880 writer.commit();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	881 } finally {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	882 if( !ok ) {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	883 writer.rollback();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	884 reopen();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	885 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	886 wrote();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	887 writeLock.unlock();
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	888 }
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	889 luanLogger.warn("end restore_from_postgres");
1388 2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	890 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	891
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	892 void restore(LuanTable doc)
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	893 throws LuanException, IOException
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	894 {
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	895 writer.addDocument(toLucene(doc,null));
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	896 }
2024d23ddd64 add restore_from_postgres Franklin Schmidt <fschmidt@gmail.com> parents: 1387 diff changeset	897
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	898 public void check(LuanFunction completer) throws IOException, SQLException, LuanException {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	899 luanLogger.info("start check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	900 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	901 if( !status.clean )
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	902 luanLogger.error("index not clean");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	903 if( postgresBackup != null )
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	904 checkPostgres(completer);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	905 luanLogger.info("end check");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	906 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	907
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	908 private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	909 final PostgresBackup.Checker postgresChecker;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	910 final IndexSearcher searcher;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	911 writeLock.lock();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	912 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	913 postgresChecker = postgresBackup.newChecker();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	914 searcher = openSearcher();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	915 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	916 writeLock.unlock();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	917 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	918 try {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	919 final List<Long> idsLucene = new ArrayList<Long>();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	920 Query query = new PrefixQuery(new Term("id"));
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	921 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	922 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	923 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	924 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	925 idsLucene.add(id);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	926 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	927 };
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	928 searcher.search(query,col);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	929 Collections.sort(idsLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	930 final List<Long> idsPostgres = postgresChecker.getIds();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	931 final int nLucene = idsLucene.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	932 final int nPostgres = idsPostgres.size();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	933 int iLucene = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	934 int iPostgres = 0;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	935 LuanToString lts = new LuanToString();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	936 lts.strict = true;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	937 lts.numberTypes = true;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	938 while( iLucene < nLucene && iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	939 long idLucene = idsLucene.get(iLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	940 long idPostgres = idsPostgres.get(iPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	941 if( idLucene < idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	942 iLucene++;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	943 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	944 } else if( idLucene > idPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	945 iPostgres++;
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	946 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	947 } else { // ==
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	948 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	949 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	950 if( td.totalHits != 1 ) throw new RuntimeException();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	951 Document doc = searcher.doc( td.scoreDocs[0].doc );
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	952 LuanTable docLucene = toTable(completer.luan(),doc);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	953 docLucene = (LuanTable)completer.call(docLucene);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	954 if( !equal(docPostgres,docLucene) ) {
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	955 luanLogger.error("id "+idLucene+" not equal");
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	956 luanLogger.error("lucene = "+lts.toString(docLucene));
cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	957 luanLogger.error("postgres = "+lts.toString(docPostgres));
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	958 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	959 iLucene++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	960 iPostgres++;
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	961 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	962 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	963 while( iLucene < nLucene ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	964 long idLucene = idsLucene.get(iLucene++);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	965 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	966 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	967 while( iPostgres < nPostgres ) {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	968 long idPostgres = idsPostgres.get(iPostgres++);
1393 cc0dbca576dc better logging Franklin Schmidt <fschmidt@gmail.com> parents: 1392 diff changeset	969 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
1391 94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	970 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	971 } finally {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	972 close(searcher);
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	973 postgresChecker.close();
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	974 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	975 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	976
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	977 private boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	978 return t1.asMap().equals(t2.asMap());
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	979 }
94f48cc76de8 add lucene check Franklin Schmidt <fschmidt@gmail.com> parents: 1390 diff changeset	980
230 4438cb2e04d0 start lucene fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> parents: diff changeset	981 }

Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1393:cc0dbca576dc