Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1267:9fa8b8389578

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
add LuanTable.luan; support metatable __gc(); add luan.sql;
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 12 Nov 2018 02:10:41 -0700
parents 475905984870
children 48f302bdc187
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
1 package luan.modules.lucene;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
2
527
Franklin Schmidt <fschmidt@gmail.com>
parents: 524
diff changeset
3 import java.io.Closeable;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
4 import java.io.File;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
6 import java.io.FileInputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
7 import java.io.IOException;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
8 import java.util.Iterator;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
9 import java.util.Map;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
10 import java.util.List;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
11 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
12 import java.util.Set;
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
13 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
14 import java.util.Collections;
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
15 import java.util.concurrent.ConcurrentMap;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
16 import java.util.concurrent.ConcurrentHashMap;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
17 import java.util.concurrent.atomic.AtomicInteger;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
18 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
19 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
20 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
21 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
22 import org.apache.lucene.analysis.Analyzer;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
23 import org.apache.lucene.analysis.core.KeywordAnalyzer;
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
24 import org.apache.lucene.document.Document;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
25 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
26 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
27 import org.apache.lucene.document.StringField;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
28 import org.apache.lucene.document.TextField;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
29 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
30 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
31 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
32 import org.apache.lucene.index.IndexableField;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
33 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
34 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
35 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
36 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
37 import org.apache.lucene.index.SnapshotDeletionPolicy;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
38 import org.apache.lucene.index.IndexCommit;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
39 import org.apache.lucene.index.AtomicReaderContext;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
40 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
41 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
42 import org.apache.lucene.util.Version;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
43 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
44 import org.apache.lucene.util.NumericUtils;
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
45 import org.apache.lucene.search.Query;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
46 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
47 import org.apache.lucene.search.TopDocs;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
48 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
49 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
50 import org.apache.lucene.search.IndexSearcher;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
51 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
52 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
53 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
54 import org.apache.lucene.search.Scorer;
620
89eb02f9827f add Lucene all_search_terms_must_match()
Franklin Schmidt <fschmidt@gmail.com>
parents: 618
diff changeset
55 import org.apache.lucene.search.BooleanClause;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
56 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
57 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
58 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
59 import org.apache.lucene.search.highlight.Fragmenter;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
60 import org.apache.lucene.search.highlight.NullFragmenter;
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
61 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
62 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
63 import org.apache.lucene.search.highlight.TokenGroup;
730
01e68da6983b add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
64 import luan.modules.lucene.queryparser.SaneQueryParser;
01e68da6983b add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
65 import luan.modules.lucene.queryparser.FieldParser;
01e68da6983b add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
66 import luan.modules.lucene.queryparser.MultiFieldParser;
01e68da6983b add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
67 import luan.modules.lucene.queryparser.StringFieldParser;
01e68da6983b add sane-lucene-queryparser source to luan
Franklin Schmidt <fschmidt@gmail.com>
parents: 707
diff changeset
68 import luan.modules.lucene.queryparser.NumberFieldParser;
1111
88b5b81cad4a move Parser to luan.lib.parser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1110
diff changeset
69 import luan.lib.parser.ParseException;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
70 import luan.modules.Utils;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
71 import luan.Luan;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
72 import luan.LuanState;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
73 import luan.LuanTable;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
74 import luan.LuanFunction;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
75 import luan.LuanException;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
76 import luan.LuanRuntimeException;
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
77 import org.slf4j.Logger;
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
78 import org.slf4j.LoggerFactory;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
79
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
80
527
Franklin Schmidt <fschmidt@gmail.com>
parents: 524
diff changeset
81 public final class LuceneIndex implements Closeable {
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
82 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
83
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
84 private static final String FLD_NEXT_ID = "nextId";
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
85 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
86
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
87 private static final Version version = Version.LUCENE_4_9;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
88 private final ReentrantLock writeLock = new ReentrantLock();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
89 private final File indexDir;
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
90 private SnapshotDeletionPolicy snapshotDeletionPolicy;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
91 private IndexWriter writer;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
92 private DirectoryReader reader;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
93 private IndexSearcher searcher;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
94 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
95 private boolean isClosed = true;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
96 private final MultiFieldParser mfp;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
97 public final LuanTable indexed_only_fields;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
98 private final Analyzer analyzer;
1227
a12dba1f0787 better logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1222
diff changeset
99 private final Exception created = new Exception("created");
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
100
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
101 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
102 private File fileDir;
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
103 private int writeCount;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
104
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
105 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
106 throws LuanException, IOException
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
107 {
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
108 indexed_only_fields = new LuanTable(luan); // not good, not thread safe
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
109 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
110 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
111 mfp.fields.put( "id", NumberFieldParser.LONG );
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
112 File indexDir = new File(indexDirStr);
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
113 this.indexDir = indexDir;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
114 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
115 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
116 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
117 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
118 }
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
119 this.analyzer = analyzer;
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
120 luan.onClose(this);
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
121 reopen();
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
122 }
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
123
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
124 public void reopen() throws LuanException, IOException {
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
125 if( !isClosed ) throw new RuntimeException();
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
126 isClosed = false;
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
127 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
128 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
129 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
130 FSDirectory dir = FSDirectory.open(indexDir);
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
131 fileDir = dir.getDirectory();
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
132 globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger());
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
133 writer = new IndexWriter(dir,conf);
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
134 writer.commit(); // commit index creation
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
135 reader = DirectoryReader.open(dir);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
136 searcher = new IndexSearcher(reader);
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
137 initId();
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
138 }
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
139
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
140 private int globalWriteCount() {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
141 return globalWriteCounters.get(fileDir).get();
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
142 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
143
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
144 private void wrote() {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
145 globalWriteCounters.get(fileDir).incrementAndGet();
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
146 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
147
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
148 public void delete_all() throws IOException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
149 boolean commit = !writeLock.isHeldByCurrentThread();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
150 writeLock.lock();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
151 try {
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
152 writer.deleteAll();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
153 id = idLim = 0;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
154 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
155 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
156 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
157 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
158 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
159 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
160
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
161 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
162 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
163 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
164 return new Term(key,br);
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
165 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
166
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
167 public void delete(LuanState luan,String queryStr) throws LuanException, IOException, ParseException {
622
1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this
Franklin Schmidt <fschmidt@gmail.com>
parents: 621
diff changeset
168 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
169
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
170 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
171 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
172 try {
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
173 writer.deleteDocuments(query);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
174 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
175 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
176 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
177 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
178 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
179 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
180
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
181 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException {
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
182 Set indexedOnlySet = new HashSet();
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
183 Object typeObj = doc.get("type");
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
184 if( typeObj==null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
185 throw new LuanException("missing 'type' field");
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
186 if( !(typeObj instanceof String) )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
187 throw new LuanException("type must be string");
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
188 String type = (String)typeObj;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
189 Object indexedOnlyObj = indexed_only_fields.get(type);
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
190 if( indexedOnlyObj != null ) {
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
191 if( !(indexedOnlyObj instanceof LuanTable) )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
192 throw new LuanException("indexed_only_fields elements must be tables");
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
193 LuanTable indexedOnly = (LuanTable)indexedOnlyObj;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
194 for( Map.Entry<Object,Object> entry : indexedOnly.iterable() ) {
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
195 Object key = entry.getKey();
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
196 if( !(key instanceof String) )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
197 throw new LuanException("indexed_only_fields."+type+" entries must be strings");
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
198 String name = (String)key;
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
199 Object value = entry.getValue();
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
200 if( !(value instanceof LuanFunction) )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
201 throw new LuanException("indexed_only_fields."+type+" values must be functions");
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
202 LuanFunction fn = (LuanFunction)value;
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
203 value = Luan.first(fn.call(luan,new Object[]{doc}));
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
204 doc.put( name, value );
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
205 indexedOnlySet.add(name);
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
206 }
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
207 }
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
208 Object obj = doc.get("id");
601
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
209 Long id;
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
210 try {
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
211 id = (Long)obj;
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
212 } catch(ClassCastException e) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
213 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
214 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
215
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
216 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
217 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
218 try {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
219 if( id == null ) {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
220 id = nextId(luan);
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
221 doc.put("id",id);
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
222 writer.addDocument(toLucene(doc,indexedOnlySet));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
223 } else {
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
224 writer.updateDocument( term("id",id), toLucene(doc,indexedOnlySet) );
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
225 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
226 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
227 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
228 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
229 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
230 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
231 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
232
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
233 public void update_in_transaction(LuanState luan,LuanFunction fn) throws IOException, LuanException {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
234 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
235 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
236 try {
574
6cc2f047019b remove LuanState.call()
Franklin Schmidt <fschmidt@gmail.com>
parents: 572
diff changeset
237 fn.call(luan);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
238 if(commit) writer.commit();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
239 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
240 wrote();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
241 writeLock.unlock();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
242 }
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
243 }
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
244
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
245 public void run_in_lock(LuanState luan,LuanFunction fn) throws IOException, LuanException {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
246 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
247 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
248 writeLock.lock();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
249 try {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
250 synchronized(this) {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
251 fn.call(luan);
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
252 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
253 } finally {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
254 wrote();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
255 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
256 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
257 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
258
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
259
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
260 private long id;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
261 private long idLim;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
262 private final int idBatch = 10;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
263
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
264 private void initId() throws LuanException, IOException {
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
265 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
266 switch(td.totalHits) {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
267 case 0:
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
268 id = 0;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
269 idLim = 0;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
270 break;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
271 case 1:
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
272 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
273 id = idLim;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
274 break;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
275 default:
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
276 throw new RuntimeException();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
277 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
278 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
279
617
e54c1646eed0 add Lucene.next_id()
Franklin Schmidt <fschmidt@gmail.com>
parents: 601
diff changeset
280 public synchronized long nextId(LuanState luan) throws LuanException, IOException {
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
281 if( ++id > idLim ) {
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
282 idLim += idBatch;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
283 LuanTable doc = new LuanTable(luan);
427
dae264ad6a7b fix LuanTable.put() to use metatables
Franklin Schmidt <fschmidt@gmail.com>
parents: 426
diff changeset
284 doc.rawPut( "type", "next_id" );
dae264ad6a7b fix LuanTable.put() to use metatables
Franklin Schmidt <fschmidt@gmail.com>
parents: 426
diff changeset
285 doc.rawPut( FLD_NEXT_ID, idLim );
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
286 writer.updateDocument(new Term("type","next_id"),toLucene(doc,Collections.EMPTY_SET));
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
287 wrote();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
288 }
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
289 return id;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
290 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
291
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
292 /*
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
293 public void backup(String zipFile) throws LuanException, IOException {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
294 if( !zipFile.endsWith(".zip") )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
295 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
296 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
297 try {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
298 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
299 for( String fileName : ic.getFileNames() ) {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
300 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
301 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
302 Utils.copyAll(in,out);
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
303 in.close();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
304 out.closeEntry();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
305 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
306 out.close();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
307 } finally {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
308 snapshotDeletionPolicy.release(ic);
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
309 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
310 }
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
311 */
1129
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
312 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
313 return snapshotDeletionPolicy;
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
314 }
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
315
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
316 public Object snapshot(LuanState luan,LuanFunction fn) throws LuanException, IOException {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
317 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
318 try {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
319 String dir = fileDir.toString();
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
320 LuanTable fileNames = new LuanTable(luan,new ArrayList(ic.getFileNames()));
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
321 return fn.call(luan,new Object[]{dir,fileNames});
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
322 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
323 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
324 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
325 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
326
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
327
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
328
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
329 public String to_string() {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
330 return writer.getDirectory().toString();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
331 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
332
303
fdb4bd391c28 add lucene close();
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 287
diff changeset
333 public void close() throws IOException {
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
334 if( !isClosed ) {
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
335 writer.close();
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
336 reader.close();
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
337 isClosed = true;
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
338 }
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
339 }
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
340
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
341 protected void finalize() throws Throwable {
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
342 if( !isClosed ) {
1227
a12dba1f0787 better logging
Franklin Schmidt <fschmidt@gmail.com>
parents: 1222
diff changeset
343 logger.error("not closed",created);
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
344 close();
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
345 }
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
346 super.finalize();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
347 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
348
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
349
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
350
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
351 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
352 final IndexSearcher searcher;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
353 int docID;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
354
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
355 DocFn(IndexSearcher searcher) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
356 this.searcher = searcher;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
357 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
358
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
359 @Override public Object call(LuanState luan,Object[] args) throws LuanException {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
360 try {
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
361 return toTable(luan,searcher.doc(docID));
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
362 } catch(IOException e) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
363 throw new LuanException(e);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
364 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
365 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
366 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
367
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
368 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
369 int docBase;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
370 int i = 0;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
371
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
372 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
373 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
374 this.docBase = context.docBase;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
375 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
376 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
377 return true;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
378 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
379 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
380
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
381 private synchronized IndexSearcher openSearcher() throws IOException {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
382 int gwc = globalWriteCount();
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
383 if( writeCount != gwc ) {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
384 writeCount = gwc;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
385 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
386 if( newReader != null ) {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
387 reader.decRef();
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
388 reader = newReader;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
389 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
390 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
391 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
392 reader.incRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
393 return searcher;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
394 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
395
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
396 // call in finally block
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
397 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
398 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
399 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
400
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
401 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
402 close(openSearcher());
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
403 }
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
404
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
405 public int advanced_search( final LuanState luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) throws LuanException, IOException, ParseException {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
406 Utils.checkNotNull(queryStr);
622
1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this
Franklin Schmidt <fschmidt@gmail.com>
parents: 621
diff changeset
407 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
408 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
409 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
410 if( !inTransaction )
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
411 searcher = openSearcher();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
412 try {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
413 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
414 if( sortStr != null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
415 throw new LuanException("sort must be nil when n is nil");
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
416 final DocFn docFn = new DocFn(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
417 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
418 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
419 try {
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
420 docFn.docID = docBase + doc;
574
6cc2f047019b remove LuanState.call()
Franklin Schmidt <fschmidt@gmail.com>
parents: 572
diff changeset
421 fn.call(luan,new Object[]{++i,docFn});
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
422 } catch(LuanException e) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
423 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
424 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
425 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
426 };
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
427 try {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
428 searcher.search(query,col);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
429 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
430 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
431 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
432 return col.i;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
433 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
434 if( fn==null || n==0 ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
435 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
436 searcher.search(query,thcc);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
437 return thcc.getTotalHits();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
438 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
439 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
440 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
441 final ScoreDoc[] scoreDocs = td.scoreDocs;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
442 DocFn docFn = new DocFn(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
443 for( int i=0; i<scoreDocs.length; i++ ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
444 docFn.docID = scoreDocs[i].doc;
574
6cc2f047019b remove LuanState.call()
Franklin Schmidt <fschmidt@gmail.com>
parents: 572
diff changeset
445 fn.call(luan,new Object[]{i+1,docFn});
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
446 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
447 return td.totalHits;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
448 } finally {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
449 if( !inTransaction )
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
450 close(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
451 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
452 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
453
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
454 public Object search_in_transaction(LuanState luan,LuanFunction fn) throws LuanException, IOException {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
455 if( threadLocalSearcher.get() != null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
456 throw new LuanException("can't nest search_in_transaction calls");
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
457 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
458 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
459 try {
574
6cc2f047019b remove LuanState.call()
Franklin Schmidt <fschmidt@gmail.com>
parents: 572
diff changeset
460 return fn.call(luan);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
461 } finally {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
462 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
463 close(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
464 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
465 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
466
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
467
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
468 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
469 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
470 }
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
471
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
472 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
473 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
474 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
475 return;
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
476 }
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
477 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
478 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
479
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
480
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
481 private IndexableField newField(String name,Object value,Field.Store store,Set<String> indexed)
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
482 throws LuanException
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
483 {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
484 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
485 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
486 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
487 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
488 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
489 return new TextField(name, s, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
490 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
491 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
492 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
493 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
494 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
495 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
496 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
497 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
498 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
499 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
500 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
501 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
502 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
503 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
504 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
505 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
506 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
507 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
508 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
509 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
510 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
511 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
512 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
513 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
514 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
515 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
516 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
517 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
518 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
519 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
520 } else
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
521 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
522 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
523
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
524 private Document toLucene(LuanTable table,Set indexOnly) throws LuanException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
525 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
526 Document doc = new Document();
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
527 for( Map.Entry<Object,Object> entry : table.iterable() ) {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
528 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
529 if( !(key instanceof String) )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
530 throw new LuanException("key must be string");
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
531 String name = (String)key;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
532 Object value = entry.getValue();
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
533 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
534 if( !(value instanceof LuanTable) ) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
535 doc.add(newField(name, value, store, indexed));
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
536 } else { // list
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
537 LuanTable list = (LuanTable)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
538 for( Object el : list.asList() ) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
539 doc.add(newField(name, el, store, indexed));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
540 }
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
541 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
542 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
543 return doc;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
544 }
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
545
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
546 private static Object getValue(IndexableField ifld) throws LuanException {
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
547 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
548 if( br != null )
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
549 return br.bytes;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
550 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
551 if( n != null )
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
552 return n;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
553 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
554 if( s != null )
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
555 return s;
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
556 throw new LuanException("invalid field type for "+ifld);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
557 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
558
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
559 private static LuanTable toTable(LuanState luan,Document doc) throws LuanException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
560 if( doc==null )
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
561 return null;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
562 LuanTable table = new LuanTable(luan);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
563 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
564 String name = ifld.name();
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
565 Object value = getValue(ifld);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
566 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
567 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
568 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
569 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
570 LuanTable list;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
571 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
572 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
573 } else {
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
574 list = new LuanTable(luan);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
575 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
576 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
577 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
578 list.rawPut(list.rawLength()+1,value);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
579 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
580 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
581 return table;
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
582 }
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
583
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
584
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
585 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
586 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
587 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
588 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
589 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
590
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
591 public LuanFunction highlighter(LuanState luan,String queryStr,LuanFunction formatter,final Integer fragmentSize,String dotdotdot) throws ParseException {
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
592 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
593 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
594 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
595 if( tokenGroup.getTotalScore() <= 0 )
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
596 return originalText;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
597 try {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
598 return (String)Luan.first(formatter.call(luan,new Object[]{originalText}));
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
599 } catch(LuanException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
600 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
601 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
602 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
603 };
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
604 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
605 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
606 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
607 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
608 final Highlighter hl = new Highlighter(fmt,queryScorer);
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
609 hl.setTextFragmenter( new NullFragmenter() );
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
610 return new LuanFunction() {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
611 @Override public String call(LuanState luan,Object[] args) throws LuanException {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
612 String text = (String)args[0];
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
613 try {
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
614 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
615 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
616 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
617 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
618 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
619 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
620 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
621 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
622 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
623 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
624 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
625 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
626 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
627 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
628 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
629 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
630 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
631 }
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
632 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
633 return s!=null ? s : text;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
634 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
635 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
636 } catch(IOException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
637 throw new RuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
638 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
639 throw new RuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
640 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
641 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
642 };
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
643 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
644 }