Mercurial Hosting > luan
annotate src/luan/modules/lucene/LuceneIndex.java @ 1528:3bd4d7963456
use goodjava/lucene/api
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 26 Jul 2020 23:11:53 -0600 |
parents | efbc3720d3f3 |
children | e6d808f40bbc |
rev | line source |
---|---|
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
1 package luan.modules.lucene; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
2 |
527 | 3 import java.io.Closeable; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
4 import java.io.File; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
5 import java.io.FileOutputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
6 import java.io.FileInputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
7 import java.io.IOException; |
1397 | 8 import java.lang.ref.Reference; |
9 import java.lang.ref.WeakReference; | |
1391 | 10 import java.sql.SQLException; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
11 import java.util.Arrays; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
12 import java.util.Iterator; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
13 import java.util.Map; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
14 import java.util.HashMap; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
15 import java.util.LinkedHashMap; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
16 import java.util.List; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
17 import java.util.ArrayList; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
18 import java.util.Set; |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
19 import java.util.HashSet; |
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
20 import java.util.Collections; |
704 | 21 import java.util.concurrent.atomic.AtomicInteger; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
22 import java.util.concurrent.locks.Lock; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
23 import java.util.concurrent.locks.ReentrantLock; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
24 import java.util.zip.ZipOutputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
25 import java.util.zip.ZipEntry; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
26 import org.apache.lucene.analysis.Analyzer; |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
27 import org.apache.lucene.analysis.TokenStream; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
28 import org.apache.lucene.analysis.core.KeywordAnalyzer; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
29 import org.apache.lucene.analysis.en.EnglishAnalyzer; |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
30 import org.apache.lucene.document.Document; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
31 import org.apache.lucene.document.Field; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
32 import org.apache.lucene.document.StoredField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
33 import org.apache.lucene.document.StringField; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
34 import org.apache.lucene.document.TextField; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
35 import org.apache.lucene.document.IntField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
36 import org.apache.lucene.document.LongField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
37 import org.apache.lucene.document.DoubleField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
38 import org.apache.lucene.index.IndexableField; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
39 import org.apache.lucene.index.IndexWriter; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
40 import org.apache.lucene.index.IndexWriterConfig; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
41 import org.apache.lucene.index.DirectoryReader; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
42 import org.apache.lucene.index.Term; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
43 import org.apache.lucene.index.SnapshotDeletionPolicy; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
44 import org.apache.lucene.index.IndexCommit; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
45 import org.apache.lucene.index.AtomicReaderContext; |
1391 | 46 import org.apache.lucene.index.CheckIndex; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
47 import org.apache.lucene.store.Directory; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
48 import org.apache.lucene.store.FSDirectory; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
49 import org.apache.lucene.util.Version; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
50 import org.apache.lucene.util.BytesRef; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
51 import org.apache.lucene.util.NumericUtils; |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
52 import org.apache.lucene.search.Query; |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
53 import org.apache.lucene.search.PrefixQuery; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
54 import org.apache.lucene.search.TermQuery; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
55 import org.apache.lucene.search.TopDocs; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
56 import org.apache.lucene.search.Sort; |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
57 import org.apache.lucene.search.SortField; |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
58 import org.apache.lucene.search.IndexSearcher; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
59 import org.apache.lucene.search.TotalHitCountCollector; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
60 import org.apache.lucene.search.ScoreDoc; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
61 import org.apache.lucene.search.Collector; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
62 import org.apache.lucene.search.Scorer; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
63 import org.apache.lucene.search.Explanation; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
64 import org.apache.lucene.search.highlight.Formatter; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
65 import org.apache.lucene.search.highlight.Highlighter; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
66 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
67 import org.apache.lucene.search.highlight.Fragmenter; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
68 import org.apache.lucene.search.highlight.NullFragmenter; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
69 import org.apache.lucene.search.highlight.SimpleSpanFragmenter; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
70 import org.apache.lucene.search.highlight.QueryScorer; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
71 import org.apache.lucene.search.highlight.TokenGroup; |
1474 | 72 import goodjava.lucene.analysis.LowercaseAnalyzer; |
1459 | 73 import goodjava.lucene.queryparser.GoodQueryParser; |
1458 | 74 import goodjava.lucene.queryparser.FieldParser; |
75 import goodjava.lucene.queryparser.MultiFieldParser; | |
76 import goodjava.lucene.queryparser.StringFieldParser; | |
77 import goodjava.lucene.queryparser.NumberFieldParser; | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
78 import goodjava.lucene.api.GoodIndexWriter; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
79 import goodjava.lucene.api.LuceneIndexWriter; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
80 import goodjava.lucene.api.GoodIndexWriterConfig; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
81 import goodjava.lucene.api.LuceneUtils; |
1402
27efb1fcbcb5
move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents:
1399
diff
changeset
|
82 import goodjava.parser.ParseException; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
83 import luan.modules.Utils; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
84 import luan.Luan; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
85 import luan.LuanTable; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
86 import luan.LuanFunction; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
87 import luan.LuanException; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
88 import luan.LuanRuntimeException; |
1391 | 89 import luan.modules.parsers.LuanToString; |
1402
27efb1fcbcb5
move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents:
1399
diff
changeset
|
90 import goodjava.logging.Logger; |
27efb1fcbcb5
move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents:
1399
diff
changeset
|
91 import goodjava.logging.LoggerFactory; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
92 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
93 |
1398 | 94 public final class LuceneIndex { |
1454 | 95 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
96 |
1397 | 97 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>(); |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
98 |
1397 | 99 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options) |
1392 | 100 throws LuanException, IOException, ClassNotFoundException, SQLException |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
101 { |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1347
diff
changeset
|
102 String key = indexDir.getCanonicalPath(); |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
103 synchronized(indexes) { |
1397 | 104 Reference<LuceneIndex> ref = indexes.get(key); |
105 if( ref != null ) { | |
106 LuceneIndex li = ref.get(); | |
1398 | 107 if( li != null ) { |
108 Object version = options.get("version"); | |
109 if( version==null || version.equals(li.version) ) | |
110 return li; | |
1397 | 111 li.closeWriter(); |
1398 | 112 } |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
113 } |
1397 | 114 LuceneIndex li = new LuceneIndex(luan,indexDir,options); |
115 indexes.put(key, new WeakReference<LuceneIndex>(li)); | |
116 return li; | |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
117 } |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
118 } |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
119 |
1398 | 120 private static final Version luceneVersion = Version.LUCENE_4_9; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
121 private static final String FLD_NEXT_ID = "nextId"; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
122 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); |
1482 | 123 // public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion)); |
1398 | 124 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion)); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
125 |
1398 | 126 private final Object version; |
1393 | 127 |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
128 private final ReentrantLock writeLock = new ReentrantLock(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
129 private final File indexDir; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
130 private GoodIndexWriter writer; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
131 private DirectoryReader reader; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
132 private IndexSearcher searcher; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
133 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
134 private final MultiFieldParser mfp; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
135 private final Analyzer analyzer; // ??? |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
136 |
1391 | 137 private FSDirectory fsDir; |
704 | 138 private int writeCount; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
139 private AtomicInteger writeCounter = new AtomicInteger(); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
140 private final GoodIndexWriterConfig config; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
141 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
142 private final PostgresBackup postgresBackup; |
1397 | 143 private boolean wasCreated; |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
144 |
1397 | 145 private LuceneIndex(Luan luan,File indexDir,LuanTable options) |
1392 | 146 throws LuanException, IOException, ClassNotFoundException, SQLException |
1267 | 147 { |
1420 | 148 options = new LuanTable(options); |
149 this.version = options.remove("version"); | |
150 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type"); | |
151 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); | |
1397 | 152 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); |
1420 | 153 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
154 LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); |
1420 | 155 Utils.checkEmpty(options); |
1397 | 156 |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
157 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
158 mfp.fields.put( "type", STRING_FIELD_PARSER ); |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
159 mfp.fields.put( "id", NumberFieldParser.LONG ); |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
160 this.indexDir = indexDir; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
161 Analyzer analyzer = STRING_FIELD_PARSER.analyzer; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
162 if( defaultFieldParser instanceof StringFieldParser ) { |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
163 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
164 analyzer = sfp.analyzer; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
165 } |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
166 this.analyzer = analyzer; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
167 this.config = new SupplementingConfig(luceneVersion,mfp,supplementer); |
1397 | 168 wasCreated = reopen(); |
1392 | 169 if( postgresSpec == null ) { |
170 postgresBackup = null; | |
171 } else { | |
1420 | 172 postgresBackup = new PostgresBackup(luan,postgresSpec); |
1397 | 173 if( !wasCreated && postgresBackup.wasCreated ) { |
1454 | 174 logger.error("rebuilding postgres backup"); |
1453 | 175 rebuild_postgres_backup(luan); |
1397 | 176 /* |
177 } else if( wasCreated && !postgresBackup.wasCreated ) { | |
1454 | 178 logger.error("restoring from postgres"); |
1397 | 179 restore_from_postgres(); |
180 */ | |
1390 | 181 } |
182 } | |
754 | 183 } |
184 | |
1390 | 185 public boolean reopen() throws IOException { |
1391 | 186 fsDir = FSDirectory.open(indexDir); |
187 boolean wasCreated = !fsDir.getDirectory().exists(); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
188 writer = new LuceneIndexWriter(fsDir,config); |
1391 | 189 reader = DirectoryReader.open(fsDir); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
190 searcher = new IndexSearcher(reader); |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
191 initId(); |
1390 | 192 return wasCreated; |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
193 } |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
194 |
704 | 195 private void wrote() { |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
196 writeCounter.incrementAndGet(); |
704 | 197 } |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
198 |
1392 | 199 public void delete_all() throws IOException, SQLException { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
200 boolean commit = !writeLock.isHeldByCurrentThread(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
201 writeLock.lock(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
202 try { |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
203 writer.deleteAll(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
204 id = idLim = 0; |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
205 if( postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
206 postgresBackup.deleteAll(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
207 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
208 } finally { |
704 | 209 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
210 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
211 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
212 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
213 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
214 private static Term term(String key,long value) { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
215 BytesRef br = new BytesRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
216 NumericUtils.longToPrefixCoded(value,0,br); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
217 return new Term(key,br); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
218 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
219 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
220 private void backupDelete(Query query) |
1392 | 221 throws IOException, SQLException, LuanException |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
222 { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
223 if( postgresBackup != null ) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
224 final List<Long> ids = new ArrayList<Long>(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
225 IndexSearcher searcher = openSearcher(); |
1391 | 226 try { |
227 MyCollector col = new MyCollector() { | |
228 @Override public void collect(int iDoc) throws IOException { | |
229 Document doc = searcher.doc( docBase + iDoc ); | |
230 Long id = (Long)doc.getField("id").numericValue(); | |
231 ids.add(id); | |
232 } | |
233 }; | |
234 searcher.search(query,col); | |
235 } finally { | |
236 close(searcher); | |
237 } | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
238 postgresBackup.begin(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
239 for( Long id : ids ) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
240 postgresBackup.delete(id); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
241 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
242 postgresBackup.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
243 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
244 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
245 |
1346 | 246 public void delete(String queryStr) |
1392 | 247 throws IOException, ParseException, SQLException, LuanException |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
248 { |
1459 | 249 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
250 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
251 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
252 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
253 try { |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
254 backupDelete(query); |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
255 writer.deleteDocuments(query); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
256 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
257 } finally { |
704 | 258 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
259 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
260 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
261 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
262 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
263 public void save(LuanTable doc) |
1392 | 264 throws LuanException, IOException, SQLException |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
265 { |
1267 | 266 Object obj = doc.get("id"); |
601 | 267 Long id; |
268 try { | |
269 id = (Long)obj; | |
270 } catch(ClassCastException e) { | |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
271 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); |
601 | 272 } |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
273 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
274 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
275 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
276 try { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
277 if( id == null ) { |
1346 | 278 id = nextId(); |
1267 | 279 doc.put("id",id); |
1391 | 280 if( postgresBackup != null ) |
281 postgresBackup.add(doc); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
282 writer.addDocument(toLucene(doc)); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
283 } else { |
1391 | 284 if( postgresBackup != null ) |
285 postgresBackup.update(doc); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
286 writer.updateDocument( "id", toLucene(doc) ); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
287 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
288 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
289 } finally { |
704 | 290 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
291 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
292 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
293 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
294 |
1392 | 295 public Object run_in_transaction(LuanFunction fn) |
296 throws IOException, LuanException, SQLException | |
297 { | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
298 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
299 writeLock.lock(); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
300 boolean ok = false; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
301 try { |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
302 if( commit && postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
303 postgresBackup.begin(); |
1379 | 304 Object rtn = fn.call(); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
305 ok = true; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
306 if(commit) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
307 if( postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
308 postgresBackup.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
309 writer.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
310 } |
1379 | 311 return rtn; |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
312 } finally { |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
313 if( !ok && commit ) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
314 if( postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
315 postgresBackup.rollback(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
316 writer.rollback(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
317 reopen(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
318 } |
704 | 319 wrote(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
320 writeLock.unlock(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
321 } |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
322 } |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
323 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
324 // ??? |
1379 | 325 public Object run_in_lock(LuanFunction fn) throws IOException, LuanException { |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
326 if( writeLock.isHeldByCurrentThread() ) |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
327 throw new RuntimeException(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
328 writeLock.lock(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
329 try { |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
330 synchronized(this) { |
1379 | 331 return fn.call(); |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
332 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
333 } finally { |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
334 wrote(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
335 writeLock.unlock(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
336 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
337 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
338 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
339 |
754 | 340 private long id; |
341 private long idLim; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
342 private final int idBatch = 10; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
343 |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
344 private void initId() throws IOException { |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
345 TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
346 switch(td.totalHits) { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
347 case 0: |
754 | 348 id = 0; |
349 idLim = 0; | |
350 break; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
351 case 1: |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
352 idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
353 id = idLim; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
354 break; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
355 default: |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
356 throw new RuntimeException(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
357 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
358 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
359 |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
360 private void saveNextId(long nextId) throws LuanException, IOException { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
361 Map<String,Object> doc = new HashMap(); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
362 doc.put( "type", "next_id" ); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
363 doc.put( FLD_NEXT_ID, idLim ); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
364 writer.updateDocument("type",doc); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
365 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
366 |
1346 | 367 public synchronized long nextId() throws LuanException, IOException { |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
368 if( ++id > idLim ) { |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
369 idLim += idBatch; |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
370 saveNextId(idLim); |
704 | 371 wrote(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
372 } |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
373 return id; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
374 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
375 |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
376 /* |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
377 public void backup(String zipFile) throws LuanException, IOException { |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
378 if( !zipFile.endsWith(".zip") ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
379 throw new LuanException("file "+zipFile+" doesn't end with '.zip'"); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
380 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
381 try { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
382 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile)); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
383 for( String fileName : ic.getFileNames() ) { |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
384 out.putNextEntry(new ZipEntry(fileName)); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
385 FileInputStream in = new FileInputStream(new File(indexDir,fileName)); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
386 Utils.copyAll(in,out); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
387 in.close(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
388 out.closeEntry(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
389 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
390 out.close(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
391 } finally { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
392 snapshotDeletionPolicy.release(ic); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
393 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
394 } |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
395 */ |
1129
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
396 public SnapshotDeletionPolicy snapshotDeletionPolicy() { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
397 return (SnapshotDeletionPolicy)writer.getLuceneConfig().getIndexDeletionPolicy(); |
1129
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
398 } |
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
399 |
1346 | 400 public Object snapshot(LuanFunction fn) throws LuanException, IOException { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
401 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
402 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
403 try { |
1391 | 404 String dir = fsDir.getDirectory().toString(); |
1346 | 405 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames())); |
1335 | 406 return fn.call(dir,fileNames); |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
407 } finally { |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
408 snapshotDeletionPolicy.release(ic); |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
409 } |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
410 } |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
411 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
412 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
413 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
414 public String to_string() { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
415 return writer.getDirectory().toString(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
416 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
417 |
1398 | 418 protected void finalize() throws Throwable { |
419 close(); | |
420 super.finalize(); | |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
421 } |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
422 |
1398 | 423 public void close() throws IOException, SQLException { |
424 closeWriter(); | |
1392 | 425 reader.close(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
426 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
427 |
1398 | 428 private void closeWriter() throws IOException, SQLException { |
1397 | 429 writeLock.lock(); |
430 try { | |
431 writer.close(); | |
1398 | 432 if( postgresBackup != null ) |
433 postgresBackup.close(); | |
1397 | 434 } finally { |
435 writeLock.unlock(); | |
436 } | |
437 } | |
438 | |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
439 |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
440 private static class DocFn extends LuanFunction { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
441 final IndexSearcher searcher; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
442 final Query query; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
443 int docID; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
444 |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
445 DocFn(Luan luan,IndexSearcher searcher,Query query) { |
1335 | 446 super(luan); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
447 this.searcher = searcher; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
448 this.query = query; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
449 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
450 |
1335 | 451 @Override public Object call(Object[] args) throws LuanException { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
452 try { |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
453 LuanTable doc = toTable(luan(),searcher.doc(docID)); |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
454 if( args.length > 0 && "explain".equals(args[0]) ) { |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
455 Explanation explanation = searcher.explain(query,docID); |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
456 return new Object[]{doc,explanation}; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
457 } else { |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
458 return doc; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
459 } |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
460 } catch(IOException e) { |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
461 throw new LuanException(e); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
462 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
463 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
464 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
465 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
466 private static abstract class MyCollector extends Collector { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
467 int docBase; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
468 int i = 0; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
469 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
470 @Override public void setScorer(Scorer scorer) {} |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
471 @Override public void setNextReader(AtomicReaderContext context) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
472 this.docBase = context.docBase; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
473 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
474 @Override public boolean acceptsDocsOutOfOrder() { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
475 return true; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
476 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
477 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
478 |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
479 private synchronized IndexSearcher openSearcher() throws IOException { |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
480 int gwc = writeCounter.get(); |
704 | 481 if( writeCount != gwc ) { |
482 writeCount = gwc; | |
483 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); | |
484 if( newReader != null ) { | |
485 reader.decRef(); | |
486 reader = newReader; | |
487 searcher = new IndexSearcher(reader); | |
488 } | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
489 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
490 reader.incRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
491 return searcher; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
492 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
493 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
494 // call in finally block |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
495 private static void close(IndexSearcher searcher) throws IOException { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
496 searcher.getIndexReader().decRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
497 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
498 |
591
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
499 public void ensure_open() throws IOException { |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
500 close(openSearcher()); |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
501 } |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
502 |
1346 | 503 public int advanced_search( String queryStr, LuanFunction fn, Integer n, String sortStr ) |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
504 throws LuanException, IOException, ParseException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
505 { |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
506 Utils.checkNotNull(queryStr); |
1459 | 507 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
508 IndexSearcher searcher = threadLocalSearcher.get(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
509 boolean inTransaction = searcher != null; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
510 if( !inTransaction ) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
511 searcher = openSearcher(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
512 try { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
513 if( fn!=null && n==null ) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
514 if( sortStr != null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
515 throw new LuanException("sort must be nil when n is nil"); |
1346 | 516 final DocFn docFn = new DocFn(fn.luan(),searcher,query); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
517 MyCollector col = new MyCollector() { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
518 @Override public void collect(int doc) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
519 try { |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
520 docFn.docID = docBase + doc; |
1335 | 521 fn.call(++i,docFn); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
522 } catch(LuanException e) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
523 throw new LuanRuntimeException(e); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
524 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
525 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
526 }; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
527 try { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
528 searcher.search(query,col); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
529 } catch(LuanRuntimeException e) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
530 throw (LuanException)e.getCause(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
531 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
532 return col.i; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
533 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
534 if( fn==null || n==0 ) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
535 TotalHitCountCollector thcc = new TotalHitCountCollector(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
536 searcher.search(query,thcc); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
537 return thcc.getTotalHits(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
538 } |
1459 | 539 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
540 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
541 final ScoreDoc[] scoreDocs = td.scoreDocs; |
1346 | 542 DocFn docFn = new DocFn(fn.luan(),searcher,query); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
543 for( int i=0; i<scoreDocs.length; i++ ) { |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
544 ScoreDoc scoreDoc = scoreDocs[i]; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
545 docFn.docID = scoreDoc.doc; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
546 fn.call(i+1,docFn,scoreDoc.score); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
547 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
548 return td.totalHits; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
549 } finally { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
550 if( !inTransaction ) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
551 close(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
552 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
553 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
554 |
1335 | 555 public Object search_in_transaction(LuanFunction fn) throws LuanException, IOException { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
556 if( threadLocalSearcher.get() != null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
557 throw new LuanException("can't nest search_in_transaction calls"); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
558 IndexSearcher searcher = openSearcher(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
559 threadLocalSearcher.set(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
560 try { |
1335 | 561 return fn.call(); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
562 } finally { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
563 threadLocalSearcher.set(null); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
564 close(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
565 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
566 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
567 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
568 |
796 | 569 public FieldParser getIndexedFieldParser(String field) { |
570 return mfp.fields.get(field); | |
571 } | |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
572 |
796 | 573 public void setIndexedFieldParser(String field,FieldParser fp) { |
574 if( fp==null ) { // delete | |
575 mfp.fields.remove(field); | |
576 return; | |
577 } | |
578 mfp.fields.put( field, fp ); | |
579 } | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
580 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
581 static Map<String,Object> toLucene(LuanTable table) throws LuanException { |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
582 Map<String,Object> map = new LinkedHashMap<String,Object>(); |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
583 for( Map.Entry<Object,Object> entry : table.iterable() ) { |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
584 String name = (String)entry.getKey(); |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
585 Object value = entry.getValue(); |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
586 if( value instanceof LuanTable ) { |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
587 LuanTable list = (LuanTable)value; |
1410 | 588 if( !list.isList() ) |
589 throw new LuanException("table value for '"+name+"' must be a list"); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
590 value = list.asList(); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
591 } |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
592 map.put(name,value); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
593 } |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
594 return map; |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
595 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
596 |
1333
25746915a241
merge Luan and LuanState
Franklin Schmidt <fschmidt@gmail.com>
parents:
1271
diff
changeset
|
597 private static LuanTable toTable(Luan luan,Document doc) throws LuanException { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
598 return doc==null ? null : toTable(luan,LuceneUtils.toMap(doc)); |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
599 } |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
600 |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
601 static LuanTable toTable(Luan luan,Map map) throws LuanException { |
1267 | 602 LuanTable table = new LuanTable(luan); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
603 for( Object obj : map.entrySet() ) { |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
604 Map.Entry entry = (Map.Entry)obj; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
605 Object value = entry.getValue(); |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
606 if( value instanceof List ) |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
607 value = new LuanTable(luan,(List)value); |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
608 table.rawPut( entry.getKey(), value ); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
609 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
610 return table; |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
611 } |
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
612 |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
613 |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
614 private static final Formatter nullFormatter = new Formatter() { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
615 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
616 return originalText; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
617 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
618 }; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
619 |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
620 public LuanFunction highlighter(String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot) |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
621 throws ParseException |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
622 { |
1459 | 623 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
624 Formatter fmt = new Formatter() { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
625 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
1435 | 626 if( tokenGroup.getTotalScore() <= 0 ) |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
627 return originalText; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
628 try { |
1335 | 629 return (String)Luan.first(formatter.call(originalText)); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
630 } catch(LuanException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
631 throw new LuanRuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
632 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
633 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
634 }; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
635 QueryScorer queryScorer = new QueryScorer(query); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
636 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
637 if( chooser != null ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
638 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
639 final Highlighter hl = new Highlighter(fmt,queryScorer); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
640 hl.setTextFragmenter( new NullFragmenter() ); |
1335 | 641 return new LuanFunction(false) { // ??? |
642 @Override public String call(Object[] args) throws LuanException { | |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
643 String text = (String)args[0]; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
644 try { |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
645 if( chooser != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
646 String s = chooser.getBestFragment(analyzer,null,text); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
647 if( s != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
648 if( dotdotdot != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
649 boolean atStart = text.startsWith(s); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
650 boolean atEnd = text.endsWith(s); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
651 if( !atStart ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
652 s = dotdotdot + s; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
653 if( !atEnd ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
654 s = s + dotdotdot; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
655 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
656 text = s; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
657 } else if( text.length() > fragmentSize ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
658 text = text.substring(0,fragmentSize); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
659 if( dotdotdot != null ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
660 text += "..."; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
661 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
662 } |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
663 String s = hl.getBestFragment(analyzer,null,text); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
664 return s!=null ? s : text; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
665 } catch(LuanRuntimeException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
666 throw (LuanException)e.getCause(); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
667 } catch(IOException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
668 throw new RuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
669 } catch(InvalidTokenOffsetsException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
670 throw new RuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
671 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
672 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
673 }; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
674 } |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
675 |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
676 public int count_tokens(String text) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
677 throws IOException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
678 { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
679 int n = 0; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
680 TokenStream ts = analyzer.tokenStream(null,text); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
681 ts.reset(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
682 while( ts.incrementToken() ) { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
683 n++; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
684 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
685 ts.close(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
686 return n; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
687 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
688 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
689 |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
690 |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
691 public boolean hasPostgresBackup() { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
692 return postgresBackup != null; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
693 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
694 |
1453 | 695 public void rebuild_postgres_backup(Luan luan) |
1392 | 696 throws IOException, LuanException, SQLException |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
697 { |
1454 | 698 logger.info("start rebuild_postgres_backup"); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
699 writeLock.lock(); |
1391 | 700 IndexSearcher searcher = openSearcher(); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
701 boolean ok = false; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
702 try { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
703 postgresBackup.begin(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
704 postgresBackup.deleteAll(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
705 Query query = new PrefixQuery(new Term("id")); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
706 MyCollector col = new MyCollector() { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
707 @Override public void collect(int iDoc) throws IOException { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
708 try { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
709 Document doc = searcher.doc( docBase + iDoc ); |
1453 | 710 LuanTable tbl = toTable(luan,doc); |
1391 | 711 postgresBackup.add(tbl); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
712 } catch(LuanException e) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
713 throw new LuanRuntimeException(e); |
1392 | 714 } catch(SQLException e) { |
715 throw new RuntimeException(e); | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
716 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
717 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
718 }; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
719 try { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
720 searcher.search(query,col); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
721 } catch(LuanRuntimeException e) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
722 throw (LuanException)e.getCause(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
723 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
724 ok = true; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
725 postgresBackup.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
726 } finally { |
1391 | 727 close(searcher); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
728 if( !ok ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
729 postgresBackup.rollback(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
730 writeLock.unlock(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
731 } |
1454 | 732 logger.info("end rebuild_postgres_backup"); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
733 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
734 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
735 public void restore_from_postgres() |
1447 | 736 throws IOException, LuanException, SQLException, ParseException |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
737 { |
1397 | 738 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) { |
1454 | 739 logger.error("restoring from postgres"); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
740 force_restore_from_postgres(); |
1397 | 741 } |
742 } | |
743 | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
744 public void force_restore_from_postgres() |
1447 | 745 throws IOException, LuanException, SQLException, ParseException |
1397 | 746 { |
1454 | 747 logger.warn("start restore_from_postgres"); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
748 if( postgresBackup==null ) |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
749 throw new NullPointerException(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
750 if( writeLock.isHeldByCurrentThread() ) |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
751 throw new RuntimeException(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
752 writeLock.lock(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
753 boolean ok = false; |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
754 try { |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
755 writer.deleteAll(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
756 long nextId = postgresBackup.maxId() + 1; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
757 postgresBackup.restoreLucene(this); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
758 id = idLim = nextId; |
1395
9dfff82dfc59
finish postgres work
Franklin Schmidt <fschmidt@gmail.com>
parents:
1393
diff
changeset
|
759 saveNextId(nextId); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
760 ok = true; |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
761 writer.commit(); |
1398 | 762 wasCreated = false; |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
763 } finally { |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
764 if( !ok ) { |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
765 writer.rollback(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
766 reopen(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
767 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
768 wrote(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
769 writeLock.unlock(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
770 } |
1454 | 771 logger.warn("end restore_from_postgres"); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
772 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
773 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
774 void restore(LuanTable doc) |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
775 throws LuanException, IOException |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
776 { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
777 writer.addDocument(toLucene(doc)); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
778 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
779 |
1453 | 780 public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException { |
1406 | 781 boolean hasPostgres = postgresBackup != null; |
782 String msg = "start check"; | |
783 if( hasPostgres ) | |
784 msg += " with postgres"; | |
1454 | 785 logger.info(msg); |
1391 | 786 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); |
787 if( !status.clean ) | |
1454 | 788 logger.error("index not clean"); |
1406 | 789 if( hasPostgres ) |
1453 | 790 checkPostgres(luan); |
1454 | 791 logger.info("end check"); |
1391 | 792 } |
793 | |
1453 | 794 private void checkPostgres(Luan luan) |
1447 | 795 throws IOException, SQLException, LuanException, ParseException |
1430 | 796 { |
1454 | 797 //logger.info("start postgres check"); |
1430 | 798 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); |
799 final IndexSearcher searcher = openSearcher(); | |
1391 | 800 try { |
801 final List<Long> idsLucene = new ArrayList<Long>(); | |
802 Query query = new PrefixQuery(new Term("id")); | |
803 MyCollector col = new MyCollector() { | |
804 @Override public void collect(int iDoc) throws IOException { | |
805 Document doc = searcher.doc( docBase + iDoc ); | |
806 Long id = (Long)doc.getField("id").numericValue(); | |
807 idsLucene.add(id); | |
808 } | |
809 }; | |
810 searcher.search(query,col); | |
811 Collections.sort(idsLucene); | |
812 final List<Long> idsPostgres = postgresChecker.getIds(); | |
813 final int nLucene = idsLucene.size(); | |
814 final int nPostgres = idsPostgres.size(); | |
815 int iLucene = 0; | |
816 int iPostgres = 0; | |
817 LuanToString lts = new LuanToString(); | |
818 lts.strict = true; | |
819 lts.numberTypes = true; | |
820 while( iLucene < nLucene && iPostgres < nPostgres ) { | |
821 long idLucene = idsLucene.get(iLucene); | |
822 long idPostgres = idsPostgres.get(iPostgres); | |
823 if( idLucene < idPostgres ) { | |
824 iLucene++; | |
1453 | 825 checkPostgres(luan,postgresChecker,lts,idLucene); |
1391 | 826 } else if( idLucene > idPostgres ) { |
827 iPostgres++; | |
1453 | 828 checkPostgres(luan,postgresChecker,lts,idPostgres); |
1391 | 829 } else { // == |
830 LuanTable docPostgres = postgresChecker.getDoc(idPostgres); | |
831 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); | |
832 if( td.totalHits != 1 ) throw new RuntimeException(); | |
833 Document doc = searcher.doc( td.scoreDocs[0].doc ); | |
1453 | 834 LuanTable docLucene = toTable(luan,doc); |
1391 | 835 if( !equal(docPostgres,docLucene) ) { |
1453 | 836 checkPostgres(luan,postgresChecker,lts,idPostgres); |
1391 | 837 } |
838 iLucene++; | |
839 iPostgres++; | |
840 } | |
841 } | |
842 while( iLucene < nLucene ) { | |
843 long idLucene = idsLucene.get(iLucene++); | |
1453 | 844 checkPostgres(luan,postgresChecker,lts,idLucene); |
1391 | 845 } |
846 while( iPostgres < nPostgres ) { | |
847 long idPostgres = idsPostgres.get(iPostgres++); | |
1453 | 848 checkPostgres(luan,postgresChecker,lts,idPostgres); |
1391 | 849 } |
850 } finally { | |
851 close(searcher); | |
852 postgresChecker.close(); | |
853 } | |
854 } | |
855 | |
1453 | 856 private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id) |
1447 | 857 throws IOException, SQLException, LuanException, ParseException |
1430 | 858 { |
1454 | 859 //logger.info("check id "+id); |
1430 | 860 writeLock.lock(); |
861 try { | |
862 final IndexSearcher searcher = openSearcher(); | |
863 try { | |
864 LuanTable docPostgres = postgresChecker.getDoc(id); | |
865 TopDocs td = searcher.search(new TermQuery(term("id",id)),1); | |
866 LuanTable docLucene; | |
867 if( td.totalHits == 0 ) { | |
868 docLucene = null; | |
869 } else if( td.totalHits == 1 ) { | |
870 Document doc = searcher.doc( td.scoreDocs[0].doc ); | |
1453 | 871 docLucene = toTable(luan,doc); |
1430 | 872 } else |
873 throw new RuntimeException(); | |
874 if( docPostgres == null ) { | |
875 if( docLucene != null ) | |
1454 | 876 logger.error("id "+id+" found in lucene but not postgres"); |
1430 | 877 return; |
878 } | |
879 if( docLucene == null ) { | |
1454 | 880 logger.error("id "+id+" found in postgres but not lucene"); |
1430 | 881 return; |
882 } | |
883 if( !equal(docPostgres,docLucene) ) { | |
1454 | 884 logger.error("id "+id+" not equal"); |
885 logger.error("lucene = "+lts.toString(docLucene)); | |
886 logger.error("postgres = "+lts.toString(docPostgres)); | |
1430 | 887 } |
888 } finally { | |
889 close(searcher); | |
890 } | |
891 } finally { | |
892 writeLock.unlock(); | |
893 } | |
894 } | |
895 | |
1453 | 896 private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException { |
897 return t1!=null && t2!=null && toJava(t1).equals(toJava(t2)); | |
1391 | 898 } |
899 | |
1453 | 900 private static Map toJava(LuanTable t) throws LuanException { |
901 Map map = t.asMap(); | |
1526 | 902 for( Iterator iter = map.entrySet().iterator(); iter.hasNext(); ) { |
903 Map.Entry entry = (Map.Entry)iter.next(); | |
1453 | 904 Object value = entry.getValue(); |
905 if( value instanceof LuanTable ) { | |
906 LuanTable v = (LuanTable)value; | |
907 if( !v.isList() ) | |
1454 | 908 logger.error("not list"); |
1526 | 909 if( v.isEmpty() ) { |
910 iter.remove(); | |
911 } else { | |
912 entry.setValue(v.asList()); | |
913 } | |
1453 | 914 } |
915 } | |
916 return map; | |
917 } | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
918 } |