Mercurial Hosting > luan

annotate src/luan/modules/lucene/LuceneIndex.java @ 1392:002152af497a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
hosted postgres
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 06 Sep 2019 00:19:47 -0600
parents 94f48cc76de8
children cc0dbca576dc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
1 package luan.modules.lucene;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
2
527
Franklin Schmidt <fschmidt@gmail.com>
parents: 524
diff changeset
3 import java.io.Closeable;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
4 import java.io.File;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
5 import java.io.FileOutputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
6 import java.io.FileInputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
7 import java.io.IOException;
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
8 import java.sql.SQLException;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
9 import java.util.Arrays;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
10 import java.util.Iterator;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
11 import java.util.Map;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
12 import java.util.HashMap;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
13 import java.util.List;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
14 import java.util.ArrayList;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
15 import java.util.Set;
618
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
16 import java.util.HashSet;
5e495e4e560b add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents: 617
diff changeset
17 import java.util.Collections;
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
18 import java.util.concurrent.atomic.AtomicInteger;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
19 import java.util.concurrent.locks.Lock;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
20 import java.util.concurrent.locks.ReentrantLock;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
21 import java.util.zip.ZipOutputStream;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
22 import java.util.zip.ZipEntry;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
23 import org.apache.lucene.analysis.Analyzer;
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
24 import org.apache.lucene.analysis.TokenStream;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
25 import org.apache.lucene.analysis.core.KeywordAnalyzer;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
26 import org.apache.lucene.analysis.en.EnglishAnalyzer;
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
27 import org.apache.lucene.document.Document;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
28 import org.apache.lucene.document.Field;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
29 import org.apache.lucene.document.StoredField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
30 import org.apache.lucene.document.StringField;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
31 import org.apache.lucene.document.TextField;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
32 import org.apache.lucene.document.IntField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
33 import org.apache.lucene.document.LongField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
34 import org.apache.lucene.document.DoubleField;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
35 import org.apache.lucene.index.IndexableField;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
36 import org.apache.lucene.index.IndexWriter;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
37 import org.apache.lucene.index.IndexWriterConfig;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
38 import org.apache.lucene.index.DirectoryReader;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
39 import org.apache.lucene.index.Term;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
40 import org.apache.lucene.index.SnapshotDeletionPolicy;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
41 import org.apache.lucene.index.IndexCommit;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
42 import org.apache.lucene.index.AtomicReaderContext;
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
43 import org.apache.lucene.index.CheckIndex;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
44 import org.apache.lucene.store.Directory;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
45 import org.apache.lucene.store.FSDirectory;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
46 import org.apache.lucene.util.Version;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
47 import org.apache.lucene.util.BytesRef;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
48 import org.apache.lucene.util.NumericUtils;
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
49 import org.apache.lucene.search.Query;
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
50 import org.apache.lucene.search.PrefixQuery;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
51 import org.apache.lucene.search.TermQuery;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
52 import org.apache.lucene.search.TopDocs;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
53 import org.apache.lucene.search.Sort;
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
54 import org.apache.lucene.search.SortField;
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
55 import org.apache.lucene.search.IndexSearcher;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
56 import org.apache.lucene.search.TotalHitCountCollector;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
57 import org.apache.lucene.search.ScoreDoc;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
58 import org.apache.lucene.search.Collector;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
59 import org.apache.lucene.search.Scorer;
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
60 import org.apache.lucene.search.Explanation;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
61 import org.apache.lucene.search.highlight.Formatter;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
62 import org.apache.lucene.search.highlight.Highlighter;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
63 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
64 import org.apache.lucene.search.highlight.Fragmenter;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
65 import org.apache.lucene.search.highlight.NullFragmenter;
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
66 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
67 import org.apache.lucene.search.highlight.QueryScorer;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
68 import org.apache.lucene.search.highlight.TokenGroup;
1344
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
69 import luan.lib.queryparser.SaneQueryParser;
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
70 import luan.lib.queryparser.FieldParser;
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
71 import luan.lib.queryparser.MultiFieldParser;
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
72 import luan.lib.queryparser.StringFieldParser;
dc2af9d5463b move queryparser to lib
Franklin Schmidt <fschmidt@gmail.com>
parents: 1343
diff changeset
73 import luan.lib.queryparser.NumberFieldParser;
1111
88b5b81cad4a move Parser to luan.lib.parser
Franklin Schmidt <fschmidt@gmail.com>
parents: 1110
diff changeset
74 import luan.lib.parser.ParseException;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
75 import luan.modules.Utils;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
76 import luan.Luan;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
77 import luan.LuanTable;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
78 import luan.LuanFunction;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
79 import luan.LuanException;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
80 import luan.LuanRuntimeException;
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
81 import luan.modules.parsers.LuanToString;
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
82 import luan.modules.logging.LuanLogger;
1337
8b61c8c4e07a remove slf4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1335
diff changeset
83 import luan.lib.logging.Logger;
8b61c8c4e07a remove slf4j
Franklin Schmidt <fschmidt@gmail.com>
parents: 1335
diff changeset
84 import luan.lib.logging.LoggerFactory;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
85
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
86
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
87 public final class LuceneIndex {
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
88 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
89
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
90 private static final class Closer implements Closeable {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
91 final LuceneIndex li;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
92 boolean isClosed = false;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
93 private final Exception created = new Exception("created");
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
94
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
95 Closer(Luan luan,LuceneIndex li) {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
96 this.li = li;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
97 luan.onClose(this);
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
98 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
99
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
100 public void close() throws IOException {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
101 if( !isClosed ) {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
102 try {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
103 li.close();
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
104 } catch(SQLException e) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
105 throw new RuntimeException(e);
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
106 }
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
107 isClosed = true;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
108 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
109 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
110
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
111 protected void finalize() throws Throwable {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
112 if( !isClosed ) {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
113 logger.error("not closed",created);
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
114 close();
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
115 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
116 super.finalize();
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
117 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
118 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
119
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
120 private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
121
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
122 public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,LuanTable postgresSpec)
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
123 throws LuanException, IOException, ClassNotFoundException, SQLException
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
124 {
1369
709f7498a363 change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1347
diff changeset
125 String key = indexDir.getCanonicalPath();
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
126 synchronized(indexes) {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
127 LuceneIndex li = indexes.get(key);
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
128 if( li == null ) {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
129 li = new LuceneIndex(luan,indexDir,defaultFieldParser,defaultFields,key,postgresSpec);
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
130 li.openCount = 1;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
131 indexes.put(key,li);
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
132 } else {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
133 if( defaultFieldParser != li.defaultFieldParser )
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
134 throw new LuanException("default_type doesn't match previous use");
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
135 if( !Arrays.equals(defaultFields,li.defaultFields) )
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
136 throw new LuanException("default_fields don't match previous use");
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
137 li.openCount++;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
138 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
139 return new Object[]{li,new Closer(luan,li)};
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
140 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
141 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
142
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
143 private static final Version version = Version.LUCENE_4_9;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
144 private static final String FLD_NEXT_ID = "nextId";
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
145 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
146 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
147
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
148 private final ReentrantLock writeLock = new ReentrantLock();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
149 private final File indexDir;
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
150 private SnapshotDeletionPolicy snapshotDeletionPolicy;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
151 private IndexWriter writer;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
152 private DirectoryReader reader;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
153 private IndexSearcher searcher;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
154 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
155 private final MultiFieldParser mfp;
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
156 private final Analyzer analyzer;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
157
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
158 private FSDirectory fsDir;
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
159 private int writeCount;
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
160 private AtomicInteger writeCounter = new AtomicInteger();
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
161
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
162 private Set<String> indexOnly = new HashSet<String>();
1271
48f302bdc187 fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents: 1267
diff changeset
163
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
164 private int openCount;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
165 private final String key;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
166 private final FieldParser defaultFieldParser;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
167 private final String[] defaultFields;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
168
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
169 private final PostgresBackup postgresBackup;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
170
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
171 private LuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanTable postgresSpec)
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
172 throws LuanException, IOException, ClassNotFoundException, SQLException
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
173 {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
174 final Logger logger = LuanLogger.getLogger(luan,LuceneIndex.class);
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
175 this.key = key;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
176 this.defaultFieldParser = defaultFieldParser;
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
177 this.defaultFields = defaultFields;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
178 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
179 mfp.fields.put( "type", STRING_FIELD_PARSER );
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
180 mfp.fields.put( "id", NumberFieldParser.LONG );
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
181 this.indexDir = indexDir;
599
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
182 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
183 if( defaultFieldParser instanceof StringFieldParser ) {
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
184 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
185 analyzer = sfp.analyzer;
50540f0813e2 support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents: 591
diff changeset
186 }
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
187 this.analyzer = analyzer;
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
188 boolean wasCreated = reopen();
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
189 if( postgresSpec == null ) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
190 postgresBackup = null;
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
191 } else {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
192 Map spec = postgresSpec.asMap();
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
193 LuanFunction completer = Utils.removeRequiredFunction(spec,"completer");
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
194 postgresBackup = new PostgresBackup(spec);
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
195 if( postgresBackup != null ) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
196 if( !wasCreated && postgresBackup.wasCreated ) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
197 logger.error("rebuilding postgres backup");
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
198 rebuild_postgres_backup(completer);
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
199 } else if( wasCreated && !postgresBackup.wasCreated ) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
200 logger.error("restoring from postgres");
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
201 restore_from_postgres(luan);
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
202 }
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
203 }
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
204 }
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
205 }
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
206
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
207 public boolean reopen() throws IOException {
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
208 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
209 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
210 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
211 fsDir = FSDirectory.open(indexDir);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
212 boolean wasCreated = !fsDir.getDirectory().exists();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
213 writer = new IndexWriter(fsDir,conf);
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
214 writer.commit(); // commit index creation
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
215 reader = DirectoryReader.open(fsDir);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
216 searcher = new IndexSearcher(reader);
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
217 initId();
1390
179c4882c6b6 backup work
Franklin Schmidt <fschmidt@gmail.com>
parents: 1388
diff changeset
218 return wasCreated;
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
219 }
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
220
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
221 private void wrote() {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
222 writeCounter.incrementAndGet();
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
223 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
224
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
225 public void delete_all() throws IOException, SQLException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
226 boolean commit = !writeLock.isHeldByCurrentThread();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
227 writeLock.lock();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
228 try {
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
229 writer.deleteAll();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
230 id = idLim = 0;
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
231 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
232 postgresBackup.deleteAll();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
233 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
234 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
235 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
236 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
237 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
238 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
239
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
240 private static Term term(String key,long value) {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
241 BytesRef br = new BytesRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
242 NumericUtils.longToPrefixCoded(value,0,br);
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
243 return new Term(key,br);
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
244 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
245
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
246 private void backupDelete(Query query)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
247 throws IOException, SQLException, LuanException
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
248 {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
249 if( postgresBackup != null ) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
250 final List<Long> ids = new ArrayList<Long>();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
251 IndexSearcher searcher = openSearcher();
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
252 try {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
253 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
254 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
255 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
256 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
257 ids.add(id);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
258 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
259 };
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
260 searcher.search(query,col);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
261 } finally {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
262 close(searcher);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
263 }
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
264 postgresBackup.begin();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
265 for( Long id : ids ) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
266 postgresBackup.delete(id);
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
267 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
268 postgresBackup.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
269 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
270 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
271
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
272 public void delete(String queryStr)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
273 throws IOException, ParseException, SQLException, LuanException
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
274 {
622
1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this
Franklin Schmidt <fschmidt@gmail.com>
parents: 621
diff changeset
275 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
276
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
277 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
278 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
279 try {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
280 backupDelete(query);
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
281 writer.deleteDocuments(query);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
282 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
283 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
284 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
285 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
286 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
287 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
288
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
289 public void indexed_only_fields(List<String> fields) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
290 indexOnly.addAll(fields);
1271
48f302bdc187 fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents: 1267
diff changeset
291 }
48f302bdc187 fix indexed_only_field
Franklin Schmidt <fschmidt@gmail.com>
parents: 1267
diff changeset
292
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
293 public void save(LuanTable doc,LuanTable boosts)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
294 throws LuanException, IOException, SQLException
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
295 {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
296 if( boosts!=null && postgresBackup!=null )
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
297 throw new LuanException("boosts are not saved to postgres backup");
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
298
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
299 Object obj = doc.get("id");
601
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
300 Long id;
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
301 try {
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
302 id = (Long)obj;
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
303 } catch(ClassCastException e) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
304 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
601
d36027b41570 better error message
Franklin Schmidt <fschmidt@gmail.com>
parents: 599
diff changeset
305 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
306
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
307 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
308 writeLock.lock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
309 try {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
310 if( id == null ) {
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
311 id = nextId();
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
312 doc.put("id",id);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
313 if( postgresBackup != null )
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
314 postgresBackup.add(doc);
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
315 writer.addDocument(toLucene(doc,boosts));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
316 } else {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
317 if( postgresBackup != null )
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
318 postgresBackup.update(doc);
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
319 writer.updateDocument( term("id",id), toLucene(doc,boosts) );
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
320 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
321 if(commit) writer.commit();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
322 } finally {
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
323 wrote();
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
324 writeLock.unlock();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
325 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
326 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
327
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
328 public Object run_in_transaction(LuanFunction fn)
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
329 throws IOException, LuanException, SQLException
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
330 {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
331 boolean commit = !writeLock.isHeldByCurrentThread();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
332 writeLock.lock();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
333 boolean ok = false;
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
334 try {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
335 if( commit && postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
336 postgresBackup.begin();
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
337 Object rtn = fn.call();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
338 ok = true;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
339 if(commit) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
340 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
341 postgresBackup.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
342 writer.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
343 }
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
344 return rtn;
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
345 } finally {
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
346 if( !ok && commit ) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
347 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
348 postgresBackup.rollback();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
349 writer.rollback();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
350 reopen();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
351 }
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
352 wrote();
252
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
353 writeLock.unlock();
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
354 }
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
355 }
3896138955b1 web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 233
diff changeset
356
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
357 // ???
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
358 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException {
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
359 if( writeLock.isHeldByCurrentThread() )
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
360 throw new RuntimeException();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
361 writeLock.lock();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
362 try {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
363 synchronized(this) {
1379
87a3738d7cc5 run_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents: 1369
diff changeset
364 return fn.call();
756
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
365 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
366 } finally {
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
367 wrote();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
368 writeLock.unlock();
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
369 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
370 }
9092e52f94eb better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 754
diff changeset
371
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
372
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
373 private long id;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
374 private long idLim;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
375 private final int idBatch = 10;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
376
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
377 private void initId() throws IOException {
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
378 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1);
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
379 switch(td.totalHits) {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
380 case 0:
754
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
381 id = 0;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
382 idLim = 0;
1a101ac9ea46 add lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents: 730
diff changeset
383 break;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
384 case 1:
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
385 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
386 id = idLim;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
387 break;
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
388 default:
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
389 throw new RuntimeException();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
390 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
391 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
392
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
393 private void saveNextId(long nextId) throws LuanException, IOException {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
394 Map doc = new HashMap();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
395 doc.put( "type", "next_id" );
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
396 doc.put( FLD_NEXT_ID, idLim );
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
397 writer.updateDocument(new Term("type","next_id"),toLucene(doc.entrySet(),null));
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
398 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
399
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
400 public synchronized long nextId() throws LuanException, IOException {
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
401 if( ++id > idLim ) {
233
ef39bc4d3f70 basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 232
diff changeset
402 idLim += idBatch;
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
403 saveNextId(idLim);
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
404 wrote();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
405 }
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
406 return id;
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
407 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
408
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
409 /*
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
410 public void backup(String zipFile) throws LuanException, IOException {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
411 if( !zipFile.endsWith(".zip") )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
412 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
413 IndexCommit ic = snapshotDeletionPolicy.snapshot();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
414 try {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
415 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
416 for( String fileName : ic.getFileNames() ) {
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
417 out.putNextEntry(new ZipEntry(fileName));
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
418 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
419 Utils.copyAll(in,out);
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
420 in.close();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
421 out.closeEntry();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
422 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
423 out.close();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
424 } finally {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
425 snapshotDeletionPolicy.release(ic);
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
426 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
427 }
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
428 */
1129
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
429 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
430 return snapshotDeletionPolicy;
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
431 }
3234a14bb1f8 minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1111
diff changeset
432
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
433 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
434 IndexCommit ic = snapshotDeletionPolicy.snapshot();
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
435 try {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
436 String dir = fsDir.getDirectory().toString();
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
437 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames()));
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
438 return fn.call(dir,fileNames);
707
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
439 } finally {
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
440 snapshotDeletionPolicy.release(ic);
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
441 }
1ed9e55f0be8 replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents: 704
diff changeset
442 }
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
443
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
444
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
445
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
446 public String to_string() {
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
447 return writer.getDirectory().toString();
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
448 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
449
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
450 private synchronized void close() throws IOException, SQLException {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
451 if( openCount > 0 ) {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
452 if( --openCount == 0 ) {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
453 doClose();
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
454 synchronized(indexes) {
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
455 indexes.remove(key);
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
456 }
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
457 }
521
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
458 }
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
459 }
8a217fe5b4f3 cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents: 427
diff changeset
460
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
461 public void doClose() throws IOException, SQLException {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
462 writer.close();
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
463 reader.close();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
464 if( postgresBackup != null )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
465 postgresBackup.close();
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
466 }
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
467
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
468
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
469 private static class DocFn extends LuanFunction {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
470 final IndexSearcher searcher;
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
471 final Query query;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
472 int docID;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
473
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
474 DocFn(Luan luan,IndexSearcher searcher,Query query) {
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
475 super(luan);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
476 this.searcher = searcher;
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
477 this.query = query;
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
478 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
479
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
480 @Override public Object call(Object[] args) throws LuanException {
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
481 try {
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
482 LuanTable doc = toTable(luan(),searcher.doc(docID));
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
483 if( args.length > 0 && "explain".equals(args[0]) ) {
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
484 Explanation explanation = searcher.explain(query,docID);
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
485 return new Object[]{doc,explanation};
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
486 } else {
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
487 return doc;
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
488 }
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
489 } catch(IOException e) {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
490 throw new LuanException(e);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
491 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
492 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
493 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
494
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
495 private static abstract class MyCollector extends Collector {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
496 int docBase;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
497 int i = 0;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
498
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
499 @Override public void setScorer(Scorer scorer) {}
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
500 @Override public void setNextReader(AtomicReaderContext context) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
501 this.docBase = context.docBase;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
502 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
503 @Override public boolean acceptsDocsOutOfOrder() {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
504 return true;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
505 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
506 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
507
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
508 private synchronized IndexSearcher openSearcher() throws IOException {
1345
6f8988830098 unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents: 1344
diff changeset
509 int gwc = writeCounter.get();
704
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
510 if( writeCount != gwc ) {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
511 writeCount = gwc;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
512 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
513 if( newReader != null ) {
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
514 reader.decRef();
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
515 reader = newReader;
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
516 searcher = new IndexSearcher(reader);
90c89138c234 optimize lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 696
diff changeset
517 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
518 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
519 reader.incRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
520 return searcher;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
521 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
522
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
523 // call in finally block
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
524 private static void close(IndexSearcher searcher) throws IOException {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
525 searcher.getIndexReader().decRef();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
526 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
527
591
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
528 public void ensure_open() throws IOException {
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
529 close(openSearcher());
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
530 }
790d5de23042 add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents: 578
diff changeset
531
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
532 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr )
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
533 throws LuanException, IOException, ParseException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
534 {
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
535 Utils.checkNotNull(queryStr);
622
1a53333eb4d5 remove Lucene all_search_terms_must_match() since now query suffix "~a" handles this
Franklin Schmidt <fschmidt@gmail.com>
parents: 621
diff changeset
536 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
537 IndexSearcher searcher = threadLocalSearcher.get();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
538 boolean inTransaction = searcher != null;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
539 if( !inTransaction )
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
540 searcher = openSearcher();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
541 try {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
542 if( fn!=null && n==null ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
543 if( sortStr != null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
544 throw new LuanException("sort must be nil when n is nil");
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
545 final DocFn docFn = new DocFn(fn.luan(),searcher,query);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
546 MyCollector col = new MyCollector() {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
547 @Override public void collect(int doc) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
548 try {
547
0be287ab0309 add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents: 546
diff changeset
549 docFn.docID = docBase + doc;
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
550 fn.call(++i,docFn);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
551 } catch(LuanException e) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
552 throw new LuanRuntimeException(e);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
553 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
554 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
555 };
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
556 try {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
557 searcher.search(query,col);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
558 } catch(LuanRuntimeException e) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
559 throw (LuanException)e.getCause();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
560 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
561 return col.i;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
562 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
563 if( fn==null || n==0 ) {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
564 TotalHitCountCollector thcc = new TotalHitCountCollector();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
565 searcher.search(query,thcc);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
566 return thcc.getTotalHits();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
567 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
568 Sort sort = sortStr==null ? null : SaneQueryParser.parseSort(mfp,sortStr);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
569 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
570 final ScoreDoc[] scoreDocs = td.scoreDocs;
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
571 DocFn docFn = new DocFn(fn.luan(),searcher,query);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
572 for( int i=0; i<scoreDocs.length; i++ ) {
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
573 ScoreDoc scoreDoc = scoreDocs[i];
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
574 docFn.docID = scoreDoc.doc;
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
575 fn.call(i+1,docFn,scoreDoc.score);
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
576 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
577 return td.totalHits;
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
578 } finally {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
579 if( !inTransaction )
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
580 close(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
581 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
582 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
583
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
584 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException {
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
585 if( threadLocalSearcher.get() != null )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
586 throw new LuanException("can't nest search_in_transaction calls");
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
587 IndexSearcher searcher = openSearcher();
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
588 threadLocalSearcher.set(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
589 try {
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
590 return fn.call();
545
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
591 } finally {
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
592 threadLocalSearcher.set(null);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
593 close(searcher);
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
594 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
595 }
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
596
ddcd4296107a clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents: 544
diff changeset
597
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
598 public FieldParser getIndexedFieldParser(String field) {
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
599 return mfp.fields.get(field);
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
600 }
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
601
796
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
602 public void setIndexedFieldParser(String field,FieldParser fp) {
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
603 if( fp==null ) { // delete
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
604 mfp.fields.remove(field);
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
605 return;
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
606 }
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
607 mfp.fields.put( field, fp );
6b8ea0a9b7c9 remove LuanMeta
Franklin Schmidt <fschmidt@gmail.com>
parents: 775
diff changeset
608 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
609
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
610
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
611 private IndexableField newField(String name,Object value,Set<String> indexed,Float boost)
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
612 throws LuanException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
613 {
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
614 boolean hasBoost = boost!=null;
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
615 IndexableField fld = newField2(name,value,indexed,hasBoost);
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
616 if( hasBoost )
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
617 ((Field)fld).setBoost(boost);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
618 return fld;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
619 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
620
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
621 private IndexableField newField2(String name,Object value,Set<String> indexed,boolean hasBoost)
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
622 throws LuanException
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
623 {
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
624 Field.Store store = indexOnly.contains(name) ? Field.Store.NO : Field.Store.YES;
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
625 if( value instanceof String ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
626 String s = (String)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
627 FieldParser fp = mfp.fields.get(name);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
628 if( fp != null ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
629 if( fp instanceof StringFieldParser && fp != STRING_FIELD_PARSER ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
630 return new TextField(name, s, store);
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
631 } else if (hasBoost) {
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
632 // fuck you modern lucene developers
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
633 return new Field(name, s, store, Field.Index.NOT_ANALYZED);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
634 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
635 return new StringField(name, s, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
636 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
637 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
638 return new StoredField(name, s);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
639 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
640 } else if( value instanceof Integer ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
641 int i = (Integer)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
642 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
643 return new IntField(name, i, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
644 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
645 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
646 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
647 } else if( value instanceof Long ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
648 long i = (Long)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
649 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
650 return new LongField(name, i, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
651 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
652 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
653 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
654 } else if( value instanceof Double ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
655 double i = (Double)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
656 if( indexed.contains(name) ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
657 return new DoubleField(name, i, store);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
658 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
659 return new StoredField(name, i);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
660 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
661 } else if( value instanceof byte[] ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
662 byte[] b = (byte[])value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
663 return new StoredField(name, b);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
664 } else
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
665 throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
666 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
667
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
668 private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
1347
643cf1c37723 move webserver to lib and bug fixes
Franklin Schmidt <fschmidt@gmail.com>
parents: 1346
diff changeset
669 return toLucene(table.iterable(),boosts);
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
670 }
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
671
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
672 private Document toLucene(Iterable<Map.Entry> iterable,LuanTable boosts) throws LuanException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
673 Set<String> indexed = mfp.fields.keySet();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
674 Document doc = new Document();
1346
Franklin Schmidt <fschmidt@gmail.com>
parents: 1345
diff changeset
675 for( Map.Entry<Object,Object> entry : iterable ) {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
676 Object key = entry.getKey();
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
677 if( !(key instanceof String) )
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
678 throw new LuanException("key must be string");
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
679 String name = (String)key;
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
680 Object value = entry.getValue();
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
681 Float boost = null;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
682 if( boosts != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
683 Object obj = boosts.get(name);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
684 if( obj != null ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
685 if( !(obj instanceof Number) )
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
686 throw new LuanException("boost '"+name+"' must be number");
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
687 boost = ((Number)obj).floatValue();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
688 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
689 }
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
690 if( !(value instanceof LuanTable) ) {
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
691 doc.add(newField( name, value, indexed, boost ));
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
692 } else { // list
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
693 LuanTable list = (LuanTable)value;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
694 for( Object el : list.asList() ) {
1343
7d9a1f8894b0 lucene change indexed_only_field() to indexed_only_fields()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1342
diff changeset
695 doc.add(newField( name, el, indexed, boost ));
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
696 }
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
697 }
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
698 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
699 return doc;
544
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
700 }
c5a93767cc5c lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents: 527
diff changeset
701
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
702 private static Object getValue(IndexableField ifld) throws LuanException {
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
703 BytesRef br = ifld.binaryValue();
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
704 if( br != null )
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
705 return br.bytes;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
706 Number n = ifld.numericValue();
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
707 if( n != null )
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
708 return n;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
709 String s = ifld.stringValue();
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
710 if( s != null )
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
711 return s;
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
712 throw new LuanException("invalid field type for "+ifld);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
713 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
714
1333
25746915a241 merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents: 1271
diff changeset
715 private static LuanTable toTable(Luan luan,Document doc) throws LuanException {
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
716 if( doc==null )
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
717 return null;
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
718 LuanTable table = new LuanTable(luan);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
719 for( IndexableField ifld : doc ) {
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
720 String name = ifld.name();
646
cdc70de628b5 simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents: 624
diff changeset
721 Object value = getValue(ifld);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
722 Object old = table.rawGet(name);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
723 if( old == null ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
724 table.rawPut(name,value);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
725 } else {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
726 LuanTable list;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
727 if( old instanceof LuanTable ) {
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
728 list = (LuanTable)old;
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
729 } else {
1267
9fa8b8389578 add LuanTable.luan;
Franklin Schmidt <fschmidt@gmail.com>
parents: 1248
diff changeset
730 list = new LuanTable(luan);
621
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
731 list.rawPut(1,old);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
732 table.rawPut(name,list);
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
733 }
fd15da41afca allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents: 620
diff changeset
734 list.rawPut(list.rawLength()+1,value);
546
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
735 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
736 }
eaef1005ab87 general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 545
diff changeset
737 return table;
312
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
738 }
d34be4588556 add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents: 303
diff changeset
739
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
740
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
741 private static final Formatter nullFormatter = new Formatter() {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
742 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
743 return originalText;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
744 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
745 };
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
746
1342
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
747 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot)
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
748 throws ParseException
60599adc27b8 add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents: 1341
diff changeset
749 {
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
750 Query query = SaneQueryParser.parseQuery(mfp,queryStr);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
751 Formatter fmt = new Formatter() {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
752 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
753 if( tokenGroup.getTotalScore() <= 0 )
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
754 return originalText;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
755 try {
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
756 return (String)Luan.first(formatter.call(originalText));
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
757 } catch(LuanException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
758 throw new LuanRuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
759 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
760 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
761 };
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
762 QueryScorer queryScorer = new QueryScorer(query);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
763 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
764 if( chooser != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
765 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) );
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
766 final Highlighter hl = new Highlighter(fmt,queryScorer);
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
767 hl.setTextFragmenter( new NullFragmenter() );
1335
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
768 return new LuanFunction(false) { // ???
e0cf0d108a77 major cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1333
diff changeset
769 @Override public String call(Object[] args) throws LuanException {
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
770 String text = (String)args[0];
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
771 try {
1248
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
772 if( chooser != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
773 String s = chooser.getBestFragment(analyzer,null,text);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
774 if( s != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
775 if( dotdotdot != null ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
776 boolean atStart = text.startsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
777 boolean atEnd = text.endsWith(s);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
778 if( !atStart )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
779 s = dotdotdot + s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
780 if( !atEnd )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
781 s = s + dotdotdot;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
782 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
783 text = s;
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
784 } else if( text.length() > fragmentSize ) {
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
785 text = text.substring(0,fragmentSize);
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
786 if( dotdotdot != null )
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
787 text += "...";
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
788 }
475905984870 improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents: 1227
diff changeset
789 }
624
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
790 String s = hl.getBestFragment(analyzer,null,text);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
791 return s!=null ? s : text;
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
792 } catch(LuanRuntimeException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
793 throw (LuanException)e.getCause();
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
794 } catch(IOException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
795 throw new RuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
796 } catch(InvalidTokenOffsetsException e) {
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
797 throw new RuntimeException(e);
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
798 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
799 }
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
800 };
8281a248c47e add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents: 622
diff changeset
801 }
1341
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
802
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
803 public int count_tokens(String text)
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
804 throws IOException
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
805 {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
806 int n = 0;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
807 TokenStream ts = analyzer.tokenStream(null,text);
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
808 ts.reset();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
809 while( ts.incrementToken() ) {
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
810 n++;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
811 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
812 ts.close();
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
813 return n;
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
814 }
a015a0b5c388 add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents: 1337
diff changeset
815
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
816
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
817
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
818 public boolean hasPostgresBackup() {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
819 return postgresBackup != null;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
820 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
821
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
822 public void rebuild_postgres_backup(LuanFunction completer)
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
823 throws IOException, LuanException, SQLException
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
824 {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
825 final Logger logger = LuanLogger.getLogger(completer.luan(),LuceneIndex.class);
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
826 logger.info("start rebuild_postgres_backup");
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
827 writeLock.lock();
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
828 IndexSearcher searcher = openSearcher();
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
829 boolean ok = false;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
830 try {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
831 postgresBackup.begin();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
832 postgresBackup.deleteAll();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
833 Query query = new PrefixQuery(new Term("id"));
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
834 MyCollector col = new MyCollector() {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
835 @Override public void collect(int iDoc) throws IOException {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
836 try {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
837 Document doc = searcher.doc( docBase + iDoc );
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
838 LuanTable tbl = toTable(completer.luan(),doc);
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
839 tbl = (LuanTable)completer.call(tbl);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
840 postgresBackup.add(tbl);
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
841 } catch(LuanException e) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
842 throw new LuanRuntimeException(e);
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
843 } catch(SQLException e) {
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
844 throw new RuntimeException(e);
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
845 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
846 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
847 };
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
848 try {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
849 searcher.search(query,col);
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
850 } catch(LuanRuntimeException e) {
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
851 throw (LuanException)e.getCause();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
852 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
853 ok = true;
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
854 postgresBackup.commit();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
855 } finally {
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
856 close(searcher);
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
857 if( !ok )
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
858 postgresBackup.rollback();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
859 writeLock.unlock();
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
860 }
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
861 logger.info("end rebuild_postgres_backup");
1387
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
862 }
bc40bc9aab3a start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents: 1379
diff changeset
863
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
864 public void restore_from_postgres(Luan luan)
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
865 throws IOException, LuanException, SQLException
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
866 {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
867 final Logger logger = LuanLogger.getLogger(luan,LuceneIndex.class);
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
868 logger.warn("start restore_from_postgres");
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
869 if( postgresBackup==null )
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
870 throw new NullPointerException();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
871 if( writeLock.isHeldByCurrentThread() )
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
872 throw new RuntimeException();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
873 writeLock.lock();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
874 boolean ok = false;
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
875 try {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
876 writer.deleteAll();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
877 long nextId = postgresBackup.maxId() + 1;
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
878 postgresBackup.restoreLucene(this);
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
879 id = idLim = nextId;
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
880 ok = true;
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
881 writer.commit();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
882 } finally {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
883 if( !ok ) {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
884 writer.rollback();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
885 reopen();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
886 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
887 wrote();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
888 writeLock.unlock();
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
889 }
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
890 logger.warn("end restore_from_postgres");
1388
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
891 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
892
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
893 void restore(LuanTable doc)
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
894 throws LuanException, IOException
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
895 {
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
896 writer.addDocument(toLucene(doc,null));
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
897 }
2024d23ddd64 add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1387
diff changeset
898
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
899 public void check(LuanFunction completer) throws IOException, SQLException, LuanException {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
900 final Logger logger = LuanLogger.getLogger(completer.luan(),LuceneIndex.class);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
901 logger.info("start check");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
902 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
903 if( !status.clean )
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
904 logger.error("index not clean");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
905 if( postgresBackup != null )
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
906 checkPostgres(completer);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
907 logger.info("end check");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
908 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
909
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
910 private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException {
1392
002152af497a hosted postgres
Franklin Schmidt <fschmidt@gmail.com>
parents: 1391
diff changeset
911 final Logger logger = LuanLogger.getLogger(completer.luan(),LuceneIndex.class);
1391
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
912 final PostgresBackup.Checker postgresChecker;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
913 final IndexSearcher searcher;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
914 writeLock.lock();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
915 try {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
916 postgresChecker = postgresBackup.newChecker();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
917 searcher = openSearcher();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
918 } finally {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
919 writeLock.unlock();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
920 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
921 try {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
922 final List<Long> idsLucene = new ArrayList<Long>();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
923 Query query = new PrefixQuery(new Term("id"));
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
924 MyCollector col = new MyCollector() {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
925 @Override public void collect(int iDoc) throws IOException {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
926 Document doc = searcher.doc( docBase + iDoc );
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
927 Long id = (Long)doc.getField("id").numericValue();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
928 idsLucene.add(id);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
929 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
930 };
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
931 searcher.search(query,col);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
932 Collections.sort(idsLucene);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
933 final List<Long> idsPostgres = postgresChecker.getIds();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
934 final int nLucene = idsLucene.size();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
935 final int nPostgres = idsPostgres.size();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
936 int iLucene = 0;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
937 int iPostgres = 0;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
938 LuanToString lts = new LuanToString();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
939 lts.strict = true;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
940 lts.numberTypes = true;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
941 while( iLucene < nLucene && iPostgres < nPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
942 long idLucene = idsLucene.get(iLucene);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
943 long idPostgres = idsPostgres.get(iPostgres);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
944 if( idLucene < idPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
945 iLucene++;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
946 logger.error("id "+idLucene+" found in lucene but not postgres");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
947 } else if( idLucene > idPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
948 iPostgres++;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
949 logger.error("id "+idPostgres+" found in postgres but not lucene");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
950 } else { // ==
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
951 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
952 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
953 if( td.totalHits != 1 ) throw new RuntimeException();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
954 Document doc = searcher.doc( td.scoreDocs[0].doc );
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
955 LuanTable docLucene = toTable(completer.luan(),doc);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
956 docLucene = (LuanTable)completer.call(docLucene);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
957 if( !equal(docPostgres,docLucene) ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
958 logger.error("id "+idLucene+" not equal");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
959 logger.error("lucene = "+lts.toString(docLucene));
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
960 logger.error("postgres = "+lts.toString(docPostgres));
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
961 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
962 iLucene++;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
963 iPostgres++;
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
964 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
965 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
966 while( iLucene < nLucene ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
967 long idLucene = idsLucene.get(iLucene++);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
968 logger.error("id "+idLucene+" found in lucene but not postgres");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
969 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
970 while( iPostgres < nPostgres ) {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
971 long idPostgres = idsPostgres.get(iPostgres++);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
972 logger.error("id "+idPostgres+" found in postgres but not lucene");
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
973 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
974 } finally {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
975 close(searcher);
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
976 postgresChecker.close();
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
977 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
978 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
979
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
980 private boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
981 return t1.asMap().equals(t2.asMap());
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
982 }
94f48cc76de8 add lucene check
Franklin Schmidt <fschmidt@gmail.com>
parents: 1390
diff changeset
983
230
4438cb2e04d0 start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff changeset
984 }