Mercurial Hosting > luan
annotate src/luan/modules/lucene/LuceneIndex.java @ 1690:973d3039c421
backup server checks client domain
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 12 Jun 2022 20:13:24 -0600 |
parents | f48db13ae2d9 |
children | 2958cf04d844 |
rev | line source |
---|---|
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
1 package luan.modules.lucene; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
2 |
527 | 3 import java.io.Closeable; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
4 import java.io.File; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
5 import java.io.FileOutputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
6 import java.io.FileInputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
7 import java.io.IOException; |
1397 | 8 import java.lang.ref.Reference; |
9 import java.lang.ref.WeakReference; | |
1391 | 10 import java.sql.SQLException; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
11 import java.util.Arrays; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
12 import java.util.Iterator; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
13 import java.util.Map; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
14 import java.util.HashMap; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
15 import java.util.List; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
16 import java.util.ArrayList; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
17 import java.util.Set; |
618
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
18 import java.util.HashSet; |
5e495e4e560b
add lucene indexed_only_fields
Franklin Schmidt <fschmidt@gmail.com>
parents:
617
diff
changeset
|
19 import java.util.Collections; |
704 | 20 import java.util.concurrent.atomic.AtomicInteger; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
21 import java.util.concurrent.locks.Lock; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
22 import java.util.concurrent.locks.ReentrantLock; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
23 import java.util.zip.ZipOutputStream; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
24 import java.util.zip.ZipEntry; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
25 import org.apache.lucene.analysis.Analyzer; |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
26 import org.apache.lucene.analysis.TokenStream; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
27 import org.apache.lucene.analysis.core.KeywordAnalyzer; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
28 import org.apache.lucene.analysis.en.EnglishAnalyzer; |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
29 import org.apache.lucene.document.Document; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
30 import org.apache.lucene.document.Field; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
31 import org.apache.lucene.document.StoredField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
32 import org.apache.lucene.document.StringField; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
33 import org.apache.lucene.document.TextField; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
34 import org.apache.lucene.document.IntField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
35 import org.apache.lucene.document.LongField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
36 import org.apache.lucene.document.DoubleField; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
37 import org.apache.lucene.index.IndexableField; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
38 import org.apache.lucene.index.IndexWriter; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
39 import org.apache.lucene.index.IndexWriterConfig; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
40 import org.apache.lucene.index.DirectoryReader; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
41 import org.apache.lucene.index.Term; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
42 import org.apache.lucene.index.SnapshotDeletionPolicy; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
43 import org.apache.lucene.index.IndexCommit; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
44 import org.apache.lucene.index.AtomicReaderContext; |
1391 | 45 import org.apache.lucene.index.CheckIndex; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
46 import org.apache.lucene.store.Directory; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
47 import org.apache.lucene.store.FSDirectory; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
48 import org.apache.lucene.util.Version; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
49 import org.apache.lucene.util.BytesRef; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
50 import org.apache.lucene.util.NumericUtils; |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
51 import org.apache.lucene.search.Query; |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
52 import org.apache.lucene.search.PrefixQuery; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
53 import org.apache.lucene.search.TermQuery; |
1539 | 54 import org.apache.lucene.search.MatchAllDocsQuery; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
55 import org.apache.lucene.search.TopDocs; |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
56 import org.apache.lucene.search.Sort; |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
57 import org.apache.lucene.search.SortField; |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
58 import org.apache.lucene.search.IndexSearcher; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
59 import org.apache.lucene.search.TotalHitCountCollector; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
60 import org.apache.lucene.search.ScoreDoc; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
61 import org.apache.lucene.search.Collector; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
62 import org.apache.lucene.search.Scorer; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
63 import org.apache.lucene.search.Explanation; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
64 import org.apache.lucene.search.highlight.Formatter; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
65 import org.apache.lucene.search.highlight.Highlighter; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
66 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
67 import org.apache.lucene.search.highlight.Fragmenter; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
68 import org.apache.lucene.search.highlight.NullFragmenter; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
69 import org.apache.lucene.search.highlight.SimpleSpanFragmenter; |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
70 import org.apache.lucene.search.highlight.QueryScorer; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
71 import org.apache.lucene.search.highlight.TokenGroup; |
1474 | 72 import goodjava.lucene.analysis.LowercaseAnalyzer; |
1459 | 73 import goodjava.lucene.queryparser.GoodQueryParser; |
1458 | 74 import goodjava.lucene.queryparser.FieldParser; |
75 import goodjava.lucene.queryparser.MultiFieldParser; | |
76 import goodjava.lucene.queryparser.StringFieldParser; | |
77 import goodjava.lucene.queryparser.NumberFieldParser; | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
78 import goodjava.lucene.api.GoodIndexWriter; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
79 import goodjava.lucene.api.LuceneIndexWriter; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
80 import goodjava.lucene.api.GoodIndexWriterConfig; |
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
81 import goodjava.lucene.api.LuceneUtils; |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
82 import goodjava.lucene.logging.LoggingIndexWriter; |
1551 | 83 import goodjava.lucene.logging.OpDoer; |
1672 | 84 import goodjava.lucene.backup.BackupIndexWriter; |
1402
27efb1fcbcb5
move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents:
1399
diff
changeset
|
85 import goodjava.parser.ParseException; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
86 import luan.modules.Utils; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
87 import luan.Luan; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
88 import luan.LuanTable; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
89 import luan.LuanFunction; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
90 import luan.LuanException; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
91 import luan.LuanRuntimeException; |
1391 | 92 import luan.modules.parsers.LuanToString; |
1402
27efb1fcbcb5
move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents:
1399
diff
changeset
|
93 import goodjava.logging.Logger; |
27efb1fcbcb5
move luan.lib to goodjava
Franklin Schmidt <fschmidt@gmail.com>
parents:
1399
diff
changeset
|
94 import goodjava.logging.LoggerFactory; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
95 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
96 |
1398 | 97 public final class LuceneIndex { |
1454 | 98 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
99 |
1397 | 100 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>(); |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
101 |
1397 | 102 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options) |
1392 | 103 throws LuanException, IOException, ClassNotFoundException, SQLException |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
104 { |
1369
709f7498a363
change Lucene.index() and add Lucene.recover()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1347
diff
changeset
|
105 String key = indexDir.getCanonicalPath(); |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
106 synchronized(indexes) { |
1397 | 107 Reference<LuceneIndex> ref = indexes.get(key); |
108 if( ref != null ) { | |
109 LuceneIndex li = ref.get(); | |
1398 | 110 if( li != null ) { |
1562 | 111 Object version = options.get(luan,"version"); |
1676 | 112 if( (version==null || version.equals(li.version)) && li.indexDir.exists() ) |
1398 | 113 return li; |
1397 | 114 li.closeWriter(); |
1398 | 115 } |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
116 } |
1397 | 117 LuceneIndex li = new LuceneIndex(luan,indexDir,options); |
118 indexes.put(key, new WeakReference<LuceneIndex>(li)); | |
119 return li; | |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
120 } |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
121 } |
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
122 |
1398 | 123 private static final Version luceneVersion = Version.LUCENE_4_9; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
124 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); |
1532
060ff7695317
Lucene.type.lowercase
Franklin Schmidt <fschmidt@gmail.com>
parents:
1530
diff
changeset
|
125 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion)); |
1398 | 126 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion)); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
127 private static final SortField ID_SORT = new SortField("id",SortField.Type.LONG); |
1539 | 128 private static final SortField ID_DESC_SORT = new SortField("id",SortField.Type.LONG,true); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
129 |
1398 | 130 private final Object version; |
1393 | 131 |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
132 private final ReentrantLock writeLock = new ReentrantLock(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
133 private final File indexDir; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
134 private GoodIndexWriter writer; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
135 private DirectoryReader reader; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
136 private IndexSearcher searcher; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
137 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
138 private final MultiFieldParser mfp; |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
139 private final Analyzer analyzer; // ??? |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
140 |
1391 | 141 private FSDirectory fsDir; |
704 | 142 private int writeCount; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
143 private AtomicInteger writeCounter = new AtomicInteger(); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
144 private final GoodIndexWriterConfig config; |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
145 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
146 private final PostgresBackup postgresBackup; |
1397 | 147 private boolean wasCreated; |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
148 private final File logDir; |
1548 | 149 private final long logTime; |
1672 | 150 private final String name; |
151 private final String domain; | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
152 |
1397 | 153 private LuceneIndex(Luan luan,File indexDir,LuanTable options) |
1392 | 154 throws LuanException, IOException, ClassNotFoundException, SQLException |
1267 | 155 { |
1420 | 156 options = new LuanTable(options); |
157 this.version = options.remove("version"); | |
158 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type"); | |
159 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); | |
1397 | 160 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); |
1420 | 161 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
162 LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
163 logDir = (File)options.remove("log_dir"); |
1548 | 164 logTime = (Long)options.remove("log_time"); |
1672 | 165 name = (String)options.remove("name"); |
1420 | 166 Utils.checkEmpty(options); |
1397 | 167 |
1672 | 168 { |
169 LuanTable module = (LuanTable)luan.require("luan:http/Http.luan"); | |
170 String domain = (String)module.get(luan,"domain"); | |
171 if( domain == null ) | |
172 domain = "localhost"; | |
173 this.domain = domain; | |
174 } | |
175 | |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
176 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
177 mfp.fields.put( "type", STRING_FIELD_PARSER ); |
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
178 mfp.fields.put( "id", NumberFieldParser.LONG ); |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
179 this.indexDir = indexDir; |
599
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
180 Analyzer analyzer = STRING_FIELD_PARSER.analyzer; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
181 if( defaultFieldParser instanceof StringFieldParser ) { |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
182 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
183 analyzer = sfp.analyzer; |
50540f0813e2
support default search fields in lucene;
Franklin Schmidt <fschmidt@gmail.com>
parents:
591
diff
changeset
|
184 } |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
185 this.analyzer = analyzer; |
1685 | 186 this.config = new SupplementingConfig(luceneVersion,mfp,supplementer); |
1397 | 187 wasCreated = reopen(); |
1392 | 188 if( postgresSpec == null ) { |
189 postgresBackup = null; | |
190 } else { | |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
191 postgresBackup = new PostgresBackup(postgresSpec); |
1397 | 192 if( !wasCreated && postgresBackup.wasCreated ) { |
1454 | 193 logger.error("rebuilding postgres backup"); |
1453 | 194 rebuild_postgres_backup(luan); |
1397 | 195 /* |
196 } else if( wasCreated && !postgresBackup.wasCreated ) { | |
1454 | 197 logger.error("restoring from postgres"); |
1397 | 198 restore_from_postgres(); |
199 */ | |
1390 | 200 } |
201 } | |
754 | 202 } |
203 | |
1390 | 204 public boolean reopen() throws IOException { |
1391 | 205 fsDir = FSDirectory.open(indexDir); |
206 boolean wasCreated = !fsDir.getDirectory().exists(); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
207 writer = new LuceneIndexWriter(fsDir,config); |
1672 | 208 if( logDir != null ) { |
209 if( BackupIndexWriter.backupDomains == null ) { | |
210 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime); | |
211 } else { | |
1690
973d3039c421
backup server checks client domain
Franklin Schmidt <fschmidt@gmail.com>
parents:
1687
diff
changeset
|
212 writer = BackupIndexWriter.newWithRestore((LuceneIndexWriter)writer,logDir,logTime,domain,name); |
1672 | 213 } |
214 } | |
1391 | 215 reader = DirectoryReader.open(fsDir); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
216 searcher = new IndexSearcher(reader); |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
217 initId(); |
1390 | 218 return wasCreated; |
233
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
219 } |
ef39bc4d3f70
basic lucene works
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
232
diff
changeset
|
220 |
704 | 221 private void wrote() { |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
222 writeCounter.incrementAndGet(); |
704 | 223 } |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
224 |
1392 | 225 public void delete_all() throws IOException, SQLException { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
226 boolean commit = !writeLock.isHeldByCurrentThread(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
227 writeLock.lock(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
228 try { |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
229 writer.deleteAll(); |
1539 | 230 id = 0; |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
231 if( postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
232 postgresBackup.deleteAll(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
233 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
234 } finally { |
704 | 235 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
236 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
237 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
238 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
239 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
240 private static Term term(String key,long value) { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
241 BytesRef br = new BytesRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
242 NumericUtils.longToPrefixCoded(value,0,br); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
243 return new Term(key,br); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
244 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
245 |
1543 | 246 private static final Set<String> ID_SET = Collections.singleton("id"); |
247 | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
248 private void backupDelete(Query query) |
1392 | 249 throws IOException, SQLException, LuanException |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
250 { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
251 if( postgresBackup != null ) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
252 final List<Long> ids = new ArrayList<Long>(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
253 IndexSearcher searcher = openSearcher(); |
1391 | 254 try { |
255 MyCollector col = new MyCollector() { | |
256 @Override public void collect(int iDoc) throws IOException { | |
1543 | 257 Document doc = searcher.doc( docBase + iDoc, ID_SET ); |
1391 | 258 Long id = (Long)doc.getField("id").numericValue(); |
259 ids.add(id); | |
260 } | |
261 }; | |
262 searcher.search(query,col); | |
263 } finally { | |
264 close(searcher); | |
265 } | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
266 postgresBackup.begin(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
267 for( Long id : ids ) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
268 postgresBackup.delete(id); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
269 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
270 postgresBackup.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
271 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
272 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
273 |
1346 | 274 public void delete(String queryStr) |
1392 | 275 throws IOException, ParseException, SQLException, LuanException |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
276 { |
1459 | 277 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
278 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
279 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
280 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
281 try { |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
282 backupDelete(query); |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
283 writer.deleteDocuments(query); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
284 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
285 } finally { |
704 | 286 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
287 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
288 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
289 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
290 |
1535 | 291 public void reindex(String queryStr) |
292 throws IOException, ParseException | |
293 { | |
294 Query query = GoodQueryParser.parseQuery(mfp,queryStr); | |
295 | |
296 boolean commit = !writeLock.isHeldByCurrentThread(); | |
297 writeLock.lock(); | |
298 try { | |
299 writer.reindexDocuments("id",query); | |
300 if(commit) writer.commit(); | |
301 } finally { | |
302 wrote(); | |
303 writeLock.unlock(); | |
304 } | |
305 } | |
306 | |
1687
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
307 public void save( Luan luan, LuanTable doc, LuanTable unstored, Map<String,Float> boosts ) |
1392 | 308 throws LuanException, IOException, SQLException |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
309 { |
1562 | 310 Object obj = doc.get(luan,"id"); |
601 | 311 Long id; |
312 try { | |
313 id = (Long)obj; | |
314 } catch(ClassCastException e) { | |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
315 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); |
601 | 316 } |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
317 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
318 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
319 writeLock.lock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
320 try { |
1687
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
321 if( unstored!=null || boosts!=null ) { |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
322 if( unstored == null ) |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
323 throw new LuanException("unstored required with boosts"); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
324 if( boosts == null ) |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
325 throw new LuanException("boosts required with unstored"); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
326 if( id != null ) |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
327 throw new LuanException("update not supported"); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
328 if( postgresBackup != null ) |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
329 throw new LuanException("not supported with postgres backup"); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
330 if( !(writer instanceof LuceneIndexWriter) ) |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
331 throw new LuanException("not supported with index logging"); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
332 id = ++this.id; |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
333 doc.put(luan,"id",id); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
334 LuceneIndexWriter liw = (LuceneIndexWriter)writer; |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
335 liw.addDocument( toLucene(doc), toLucene(unstored), boosts ); |
f48db13ae2d9
unlogged lucene support
Franklin Schmidt <fschmidt@gmail.com>
parents:
1685
diff
changeset
|
336 } else if( id == null ) { |
1539 | 337 id = ++this.id; |
1562 | 338 doc.put(luan,"id",id); |
1391 | 339 if( postgresBackup != null ) |
1562 | 340 postgresBackup.add(luan,doc); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
341 writer.addDocument(toLucene(doc)); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
342 } else { |
1391 | 343 if( postgresBackup != null ) |
1562 | 344 postgresBackup.update(luan,doc); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
345 writer.updateDocument( "id", toLucene(doc) ); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
346 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
347 if(commit) writer.commit(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
348 } finally { |
704 | 349 wrote(); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
350 writeLock.unlock(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
351 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
352 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
353 |
1533
28f06ac6ce39
add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents:
1532
diff
changeset
|
354 public boolean is_in_transaction() { |
28f06ac6ce39
add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents:
1532
diff
changeset
|
355 return writeLock.isHeldByCurrentThread(); |
28f06ac6ce39
add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents:
1532
diff
changeset
|
356 } |
28f06ac6ce39
add not_in_transaction
Franklin Schmidt <fschmidt@gmail.com>
parents:
1532
diff
changeset
|
357 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
358 public Object run_in_transaction(Luan luan,LuanFunction fn) |
1392 | 359 throws IOException, LuanException, SQLException |
360 { | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
361 boolean commit = !writeLock.isHeldByCurrentThread(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
362 writeLock.lock(); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
363 boolean ok = false; |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
364 try { |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
365 if( commit && postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
366 postgresBackup.begin(); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
367 Object rtn = fn.call(luan); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
368 ok = true; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
369 if(commit) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
370 if( postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
371 postgresBackup.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
372 writer.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
373 } |
1379 | 374 return rtn; |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
375 } finally { |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
376 if( !ok && commit ) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
377 if( postgresBackup != null ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
378 postgresBackup.rollback(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
379 writer.rollback(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
380 reopen(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
381 } |
704 | 382 wrote(); |
252
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
383 writeLock.unlock(); |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
384 } |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
385 } |
3896138955b1
web testing...
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
233
diff
changeset
|
386 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
387 // ??? |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
388 public Object run_in_lock(Luan luan,LuanFunction fn) throws IOException, LuanException { |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
389 if( writeLock.isHeldByCurrentThread() ) |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
390 throw new RuntimeException(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
391 writeLock.lock(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
392 try { |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
393 synchronized(this) { |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
394 return fn.call(luan); |
756
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
395 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
396 } finally { |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
397 wrote(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
398 writeLock.unlock(); |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
399 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
400 } |
9092e52f94eb
better synchronization for lucene restore
Franklin Schmidt <fschmidt@gmail.com>
parents:
754
diff
changeset
|
401 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
402 |
754 | 403 private long id; |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
404 |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
405 private void initId() throws IOException { |
1539 | 406 TopDocs td = searcher.search(new MatchAllDocsQuery(),1,new Sort(ID_DESC_SORT)); |
407 switch(td.scoreDocs.length) { | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
408 case 0: |
754 | 409 id = 0; |
410 break; | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
411 case 1: |
1539 | 412 id = (Long)searcher.doc(td.scoreDocs[0].doc).getField("id").numericValue(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
413 break; |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
414 default: |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
415 throw new RuntimeException(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
416 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
417 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
418 |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
419 /* |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
420 public void backup(String zipFile) throws LuanException, IOException { |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
421 if( !zipFile.endsWith(".zip") ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
422 throw new LuanException("file "+zipFile+" doesn't end with '.zip'"); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
423 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
424 try { |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
425 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile)); |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
426 for( String fileName : ic.getFileNames() ) { |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
427 out.putNextEntry(new ZipEntry(fileName)); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
428 FileInputStream in = new FileInputStream(new File(indexDir,fileName)); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
429 Utils.copyAll(in,out); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
430 in.close(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
431 out.closeEntry(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
432 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
433 out.close(); |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
434 } finally { |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
435 snapshotDeletionPolicy.release(ic); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
436 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
437 } |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
438 */ |
1129
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
439 public SnapshotDeletionPolicy snapshotDeletionPolicy() { |
1539 | 440 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy(); |
1129
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
441 } |
3234a14bb1f8
minor lucene changes
Franklin Schmidt <fschmidt@gmail.com>
parents:
1111
diff
changeset
|
442 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
443 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
444 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
445 IndexCommit ic = snapshotDeletionPolicy.snapshot(); |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
446 try { |
1391 | 447 String dir = fsDir.getDirectory().toString(); |
1562 | 448 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames())); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
449 return fn.call(luan,dir,fileNames); |
707
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
450 } finally { |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
451 snapshotDeletionPolicy.release(ic); |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
452 } |
1ed9e55f0be8
replace lucene.backup with lucene.zip implemented in luan, and add lucene.snapshot
Franklin Schmidt <fschmidt@gmail.com>
parents:
704
diff
changeset
|
453 } |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
454 |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
455 public void tag(String tag) throws IOException { |
1556 | 456 boolean commit = !writeLock.isHeldByCurrentThread(); |
457 writeLock.lock(); | |
458 try { | |
459 writer.tag(tag); | |
460 if(commit) writer.commit(); | |
461 } finally { | |
462 writeLock.unlock(); | |
463 } | |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
464 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
465 |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
466 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
467 |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
468 public String to_string() { |
1539 | 469 return writer.getLuceneIndexWriter().getDirectory().toString(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
470 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
471 |
1398 | 472 protected void finalize() throws Throwable { |
473 close(); | |
474 super.finalize(); | |
521
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
475 } |
8a217fe5b4f3
cleaner LuanState.onClose()
Franklin Schmidt <fschmidt@gmail.com>
parents:
427
diff
changeset
|
476 |
1398 | 477 public void close() throws IOException, SQLException { |
478 closeWriter(); | |
1392 | 479 reader.close(); |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
480 } |
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
481 |
1398 | 482 private void closeWriter() throws IOException, SQLException { |
1397 | 483 writeLock.lock(); |
484 try { | |
485 writer.close(); | |
1398 | 486 if( postgresBackup != null ) |
487 postgresBackup.close(); | |
1397 | 488 } finally { |
489 writeLock.unlock(); | |
490 } | |
491 } | |
492 | |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
493 |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
494 private static class DocFn extends LuanFunction { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
495 final IndexSearcher searcher; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
496 final Query query; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
497 int docID; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
498 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
499 DocFn(IndexSearcher searcher,Query query) { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
500 this.searcher = searcher; |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
501 this.query = query; |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
502 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
503 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
504 @Override public Object call(Luan luan,Object[] args) throws LuanException { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
505 try { |
1562 | 506 LuanTable doc = toTable(searcher.doc(docID)); |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
507 if( args.length > 0 && "explain".equals(args[0]) ) { |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
508 Explanation explanation = searcher.explain(query,docID); |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
509 return new Object[]{doc,explanation}; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
510 } else { |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
511 return doc; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
512 } |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
513 } catch(IOException e) { |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
514 throw new LuanException(e); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
515 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
516 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
517 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
518 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
519 private static abstract class MyCollector extends Collector { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
520 int docBase; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
521 int i = 0; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
522 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
523 @Override public void setScorer(Scorer scorer) {} |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
524 @Override public void setNextReader(AtomicReaderContext context) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
525 this.docBase = context.docBase; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
526 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
527 @Override public boolean acceptsDocsOutOfOrder() { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
528 return true; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
529 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
530 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
531 |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
532 private synchronized IndexSearcher openSearcher() throws IOException { |
1345
6f8988830098
unique LuceneIndex per dir
Franklin Schmidt <fschmidt@gmail.com>
parents:
1344
diff
changeset
|
533 int gwc = writeCounter.get(); |
704 | 534 if( writeCount != gwc ) { |
535 writeCount = gwc; | |
536 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); | |
1539 | 537 // DirectoryReader newReader = DirectoryReader.openIfChanged(reader,writer.getLuceneIndexWriter(),true); |
704 | 538 if( newReader != null ) { |
539 reader.decRef(); | |
540 reader = newReader; | |
541 searcher = new IndexSearcher(reader); | |
542 } | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
543 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
544 reader.incRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
545 return searcher; |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
546 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
547 |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
548 // call in finally block |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
549 private static void close(IndexSearcher searcher) throws IOException { |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
550 searcher.getIndexReader().decRef(); |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
551 } |
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
552 |
591
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
553 public void ensure_open() throws IOException { |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
554 close(openSearcher()); |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
555 } |
790d5de23042
add "strict" param to Io.repr();
Franklin Schmidt <fschmidt@gmail.com>
parents:
578
diff
changeset
|
556 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
557 public int advanced_search( final Luan luan, String queryStr, LuanFunction fn, Integer n, String sortStr ) |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
558 throws LuanException, IOException, ParseException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
559 { |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
560 Utils.checkNotNull(queryStr); |
1459 | 561 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
562 IndexSearcher searcher = threadLocalSearcher.get(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
563 boolean inTransaction = searcher != null; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
564 if( !inTransaction ) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
565 searcher = openSearcher(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
566 try { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
567 if( fn!=null && n==null ) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
568 if( sortStr != null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
569 throw new LuanException("sort must be nil when n is nil"); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
570 final DocFn docFn = new DocFn(searcher,query); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
571 MyCollector col = new MyCollector() { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
572 @Override public void collect(int doc) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
573 try { |
547
0be287ab0309
add lucene/Versioning and simplify Lucene fn names
Franklin Schmidt <fschmidt@gmail.com>
parents:
546
diff
changeset
|
574 docFn.docID = docBase + doc; |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
575 fn.call(luan,++i,docFn); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
576 } catch(LuanException e) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
577 throw new LuanRuntimeException(e); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
578 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
579 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
580 }; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
581 try { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
582 searcher.search(query,col); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
583 } catch(LuanRuntimeException e) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
584 throw (LuanException)e.getCause(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
585 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
586 return col.i; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
587 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
588 if( fn==null || n==0 ) { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
589 TotalHitCountCollector thcc = new TotalHitCountCollector(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
590 searcher.search(query,thcc); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
591 return thcc.getTotalHits(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
592 } |
1459 | 593 Sort sort = sortStr==null ? null : GoodQueryParser.parseSort(mfp,sortStr); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
594 TopDocs td = sort==null ? searcher.search(query,n) : searcher.search(query,n,sort); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
595 final ScoreDoc[] scoreDocs = td.scoreDocs; |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
596 DocFn docFn = new DocFn(searcher,query); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
597 for( int i=0; i<scoreDocs.length; i++ ) { |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
598 ScoreDoc scoreDoc = scoreDocs[i]; |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
599 docFn.docID = scoreDoc.doc; |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
600 fn.call(luan,i+1,docFn,scoreDoc.score); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
601 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
602 return td.totalHits; |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
603 } finally { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
604 if( !inTransaction ) |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
605 close(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
606 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
607 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
608 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
609 public Object search_in_transaction(Luan luan,LuanFunction fn) throws LuanException, IOException { |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
610 if( threadLocalSearcher.get() != null ) |
646
cdc70de628b5
simplify LuanException
Franklin Schmidt <fschmidt@gmail.com>
parents:
624
diff
changeset
|
611 throw new LuanException("can't nest search_in_transaction calls"); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
612 IndexSearcher searcher = openSearcher(); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
613 threadLocalSearcher.set(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
614 try { |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
615 return fn.call(luan); |
545
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
616 } finally { |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
617 threadLocalSearcher.set(null); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
618 close(searcher); |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
619 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
620 } |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
621 |
ddcd4296107a
clean up lucene search
Franklin Schmidt <fschmidt@gmail.com>
parents:
544
diff
changeset
|
622 |
796 | 623 public FieldParser getIndexedFieldParser(String field) { |
624 return mfp.fields.get(field); | |
625 } | |
544
c5a93767cc5c
lucene overhaul, untested
Franklin Schmidt <fschmidt@gmail.com>
parents:
527
diff
changeset
|
626 |
796 | 627 public void setIndexedFieldParser(String field,FieldParser fp) { |
628 if( fp==null ) { // delete | |
629 mfp.fields.remove(field); | |
630 return; | |
631 } | |
632 mfp.fields.put( field, fp ); | |
633 } | |
546
eaef1005ab87
general lucene cleanup
Franklin Schmidt <fschmidt@gmail.com>
parents:
545
diff
changeset
|
634 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
635 static Map<String,Object> toLucene(LuanTable table) throws LuanException { |
1529 | 636 return SupplementingConfig.toLucene(table); |
621
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
637 } |
fd15da41afca
allow list of values to be stored in lucene
Franklin Schmidt <fschmidt@gmail.com>
parents:
620
diff
changeset
|
638 |
1562 | 639 private static LuanTable toTable(Document doc) throws LuanException { |
640 return doc==null ? null : SupplementingConfig.toTable(LuceneUtils.toMap(doc)); | |
312
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
641 } |
d34be4588556
add lucene query parsing
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
303
diff
changeset
|
642 |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
643 |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
644 private static final Formatter nullFormatter = new Formatter() { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
645 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
646 return originalText; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
647 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
648 }; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
649 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
650 public LuanFunction highlighter(final Luan luan,String queryStr,final LuanFunction formatter,final Integer fragmentSize,String dotdotdot) |
1342
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
651 throws ParseException |
60599adc27b8
add lucene search options
Franklin Schmidt <fschmidt@gmail.com>
parents:
1341
diff
changeset
|
652 { |
1459 | 653 Query query = GoodQueryParser.parseQuery(mfp,queryStr); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
654 Formatter fmt = new Formatter() { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
655 public String highlightTerm(String originalText,TokenGroup tokenGroup) { |
1435 | 656 if( tokenGroup.getTotalScore() <= 0 ) |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
657 return originalText; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
658 try { |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
659 return (String)Luan.first(formatter.call(luan,originalText)); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
660 } catch(LuanException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
661 throw new LuanRuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
662 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
663 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
664 }; |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
665 QueryScorer queryScorer = new QueryScorer(query); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
666 final Highlighter chooser = fragmentSize==null ? null : new Highlighter(nullFormatter,queryScorer); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
667 if( chooser != null ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
668 chooser.setTextFragmenter( new SimpleSpanFragmenter(queryScorer,fragmentSize) ); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
669 final Highlighter hl = new Highlighter(fmt,queryScorer); |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
670 hl.setTextFragmenter( new NullFragmenter() ); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
671 return new LuanFunction() { |
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
672 @Override public String call(Luan luan,Object[] args) throws LuanException { |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
673 String text = (String)args[0]; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
674 try { |
1248
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
675 if( chooser != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
676 String s = chooser.getBestFragment(analyzer,null,text); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
677 if( s != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
678 if( dotdotdot != null ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
679 boolean atStart = text.startsWith(s); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
680 boolean atEnd = text.endsWith(s); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
681 if( !atStart ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
682 s = dotdotdot + s; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
683 if( !atEnd ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
684 s = s + dotdotdot; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
685 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
686 text = s; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
687 } else if( text.length() > fragmentSize ) { |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
688 text = text.substring(0,fragmentSize); |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
689 if( dotdotdot != null ) |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
690 text += "..."; |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
691 } |
475905984870
improve lucene highlighter and allow bbcode_to_text quoter to be nil
Franklin Schmidt <fschmidt@gmail.com>
parents:
1227
diff
changeset
|
692 } |
624
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
693 String s = hl.getBestFragment(analyzer,null,text); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
694 return s!=null ? s : text; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
695 } catch(LuanRuntimeException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
696 throw (LuanException)e.getCause(); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
697 } catch(IOException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
698 throw new RuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
699 } catch(InvalidTokenOffsetsException e) { |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
700 throw new RuntimeException(e); |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
701 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
702 } |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
703 }; |
8281a248c47e
add lucene highlighter
Franklin Schmidt <fschmidt@gmail.com>
parents:
622
diff
changeset
|
704 } |
1341
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
705 |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
706 public int count_tokens(String text) |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
707 throws IOException |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
708 { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
709 int n = 0; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
710 TokenStream ts = analyzer.tokenStream(null,text); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
711 ts.reset(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
712 while( ts.incrementToken() ) { |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
713 n++; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
714 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
715 ts.close(); |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
716 return n; |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
717 } |
a015a0b5c388
add Html.decode(), Lucene.count_tokens(), lucene boosts, Sql.database.set()
Franklin Schmidt <fschmidt@gmail.com>
parents:
1337
diff
changeset
|
718 |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
719 |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
720 |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
721 public boolean hasPostgresBackup() { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
722 return postgresBackup != null; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
723 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
724 |
1453 | 725 public void rebuild_postgres_backup(Luan luan) |
1392 | 726 throws IOException, LuanException, SQLException |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
727 { |
1454 | 728 logger.info("start rebuild_postgres_backup"); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
729 writeLock.lock(); |
1391 | 730 IndexSearcher searcher = openSearcher(); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
731 boolean ok = false; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
732 try { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
733 postgresBackup.begin(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
734 postgresBackup.deleteAll(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
735 Query query = new PrefixQuery(new Term("id")); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
736 MyCollector col = new MyCollector() { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
737 @Override public void collect(int iDoc) throws IOException { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
738 try { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
739 Document doc = searcher.doc( docBase + iDoc ); |
1562 | 740 LuanTable tbl = toTable(doc); |
741 postgresBackup.add(luan,tbl); | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
742 } catch(LuanException e) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
743 throw new LuanRuntimeException(e); |
1392 | 744 } catch(SQLException e) { |
745 throw new RuntimeException(e); | |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
746 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
747 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
748 }; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
749 try { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
750 searcher.search(query,col); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
751 } catch(LuanRuntimeException e) { |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
752 throw (LuanException)e.getCause(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
753 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
754 ok = true; |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
755 postgresBackup.commit(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
756 } finally { |
1391 | 757 close(searcher); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
758 if( !ok ) |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
759 postgresBackup.rollback(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
760 writeLock.unlock(); |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
761 } |
1454 | 762 logger.info("end rebuild_postgres_backup"); |
1387
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
763 } |
bc40bc9aab3a
start postgres backup
Franklin Schmidt <fschmidt@gmail.com>
parents:
1379
diff
changeset
|
764 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
765 public void restore_from_postgres() |
1447 | 766 throws IOException, LuanException, SQLException, ParseException |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
767 { |
1397 | 768 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) { |
1454 | 769 logger.error("restoring from postgres"); |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
770 force_restore_from_postgres(); |
1397 | 771 } |
772 } | |
773 | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
774 public void force_restore_from_postgres() |
1447 | 775 throws IOException, LuanException, SQLException, ParseException |
1397 | 776 { |
1454 | 777 logger.warn("start restore_from_postgres"); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
778 if( postgresBackup==null ) |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
779 throw new NullPointerException(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
780 if( writeLock.isHeldByCurrentThread() ) |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
781 throw new RuntimeException(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
782 writeLock.lock(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
783 boolean ok = false; |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
784 try { |
1548 | 785 writer.tag("restore_from_postgres"); |
786 writer.deleteAll(); | |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
787 postgresBackup.restoreLucene(this); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
788 ok = true; |
1548 | 789 writer.commit(); |
1539 | 790 wrote(); |
791 ensure_open(); // refresh searcher | |
792 initId(); | |
1398 | 793 wasCreated = false; |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
794 } finally { |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
795 if( !ok ) { |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
796 writer.rollback(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
797 reopen(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
798 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
799 wrote(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
800 writeLock.unlock(); |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
801 } |
1454 | 802 logger.warn("end restore_from_postgres"); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
803 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
804 |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
805 void restore(LuanTable doc) |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
806 throws LuanException, IOException |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
807 { |
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1526
diff
changeset
|
808 writer.addDocument(toLucene(doc)); |
1388
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
809 } |
2024d23ddd64
add restore_from_postgres
Franklin Schmidt <fschmidt@gmail.com>
parents:
1387
diff
changeset
|
810 |
1556 | 811 public void relog() |
812 throws IOException, LuanException | |
813 { | |
814 logger.info("start relog"); | |
815 writeLock.lock(); | |
816 try { | |
817 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; | |
818 loggingWriter.logLucene(); | |
819 } finally { | |
820 writeLock.unlock(); | |
821 } | |
822 logger.info("end relog"); | |
823 } | |
824 | |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
825 public void restore_from_log(Luan luan,LuanFunction handler) |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
826 throws IOException, LuanException, SQLException, ParseException |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
827 { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
828 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
829 if( wasCreated && !loggingWriter.wasCreated ) { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
830 logger.error("restoring from log"); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
831 force_restore_from_log(luan,handler); |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
832 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
833 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
834 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
835 public void force_restore_from_log(Luan luan,LuanFunction handler) |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
836 throws IOException |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
837 { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
838 logger.warn("start force_restore_from_log"); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
839 if( writeLock.isHeldByCurrentThread() ) |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
840 throw new RuntimeException(); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
841 OpDoer opDoer = handler==null ? null : new LuanOpDoer(writer,luan,handler); |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
842 writeLock.lock(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
843 boolean ok = false; |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
844 try { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
845 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; |
1551 | 846 loggingWriter.playLogs(opDoer); |
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
847 ok = true; |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
848 wrote(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
849 ensure_open(); // refresh searcher |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
850 initId(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
851 wasCreated = false; |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
852 } finally { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
853 if( !ok ) { |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
854 writer.rollback(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
855 reopen(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
856 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
857 wrote(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
858 writeLock.unlock(); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
859 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
860 logger.warn("end force_restore_from_log"); |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
861 } |
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1543
diff
changeset
|
862 |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
863 public void check() throws IOException, SQLException, LuanException, ParseException { |
1406 | 864 boolean hasPostgres = postgresBackup != null; |
865 String msg = "start check"; | |
866 if( hasPostgres ) | |
867 msg += " with postgres"; | |
1454 | 868 logger.info(msg); |
1391 | 869 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); |
870 if( !status.clean ) | |
1454 | 871 logger.error("index not clean"); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
872 if( writer instanceof LoggingIndexWriter ) { |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
873 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
874 logger.info("log check"); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
875 boolean ok = loggingWriter.check(ID_SORT); |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
876 } |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
877 if( hasPostgres ) { |
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
878 logger.info("postgres check"); |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
879 checkPostgres(); |
1538
634f6765830e
use goodjava/lucene/logging
Franklin Schmidt <fschmidt@gmail.com>
parents:
1535
diff
changeset
|
880 } |
1454 | 881 logger.info("end check"); |
1391 | 882 } |
883 | |
1563
8fbcc4747091
remove LuanFunction.luan
Franklin Schmidt <fschmidt@gmail.com>
parents:
1562
diff
changeset
|
884 private void checkPostgres() |
1447 | 885 throws IOException, SQLException, LuanException, ParseException |
1430 | 886 { |
887 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); | |
888 final IndexSearcher searcher = openSearcher(); | |
1391 | 889 try { |
890 final List<Long> idsLucene = new ArrayList<Long>(); | |
891 Query query = new PrefixQuery(new Term("id")); | |
892 MyCollector col = new MyCollector() { | |
893 @Override public void collect(int iDoc) throws IOException { | |
894 Document doc = searcher.doc( docBase + iDoc ); | |
895 Long id = (Long)doc.getField("id").numericValue(); | |
896 idsLucene.add(id); | |
897 } | |
898 }; | |
899 searcher.search(query,col); | |
900 Collections.sort(idsLucene); | |
901 final List<Long> idsPostgres = postgresChecker.getIds(); | |
902 final int nLucene = idsLucene.size(); | |
903 final int nPostgres = idsPostgres.size(); | |
904 int iLucene = 0; | |
905 int iPostgres = 0; | |
1623
582384548a69
LuanToString takes function
Franklin Schmidt <fschmidt@gmail.com>
parents:
1563
diff
changeset
|
906 LuanToString lts = new LuanToString(); |
1553 | 907 lts.settingsInit.strict = true; |
908 lts.settingsInit.numberTypes = true; | |
1391 | 909 while( iLucene < nLucene && iPostgres < nPostgres ) { |
910 long idLucene = idsLucene.get(iLucene); | |
911 long idPostgres = idsPostgres.get(iPostgres); | |
912 if( idLucene < idPostgres ) { | |
913 iLucene++; | |
1562 | 914 checkPostgres(postgresChecker,lts,idLucene); |
1391 | 915 } else if( idLucene > idPostgres ) { |
916 iPostgres++; | |
1562 | 917 checkPostgres(postgresChecker,lts,idPostgres); |
1391 | 918 } else { // == |
919 LuanTable docPostgres = postgresChecker.getDoc(idPostgres); | |
920 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); | |
921 if( td.totalHits != 1 ) throw new RuntimeException(); | |
922 Document doc = searcher.doc( td.scoreDocs[0].doc ); | |
1562 | 923 LuanTable docLucene = toTable(doc); |
1391 | 924 if( !equal(docPostgres,docLucene) ) { |
1562 | 925 checkPostgres(postgresChecker,lts,idPostgres); |
1391 | 926 } |
927 iLucene++; | |
928 iPostgres++; | |
929 } | |
930 } | |
931 while( iLucene < nLucene ) { | |
932 long idLucene = idsLucene.get(iLucene++); | |
1562 | 933 checkPostgres(postgresChecker,lts,idLucene); |
1391 | 934 } |
935 while( iPostgres < nPostgres ) { | |
936 long idPostgres = idsPostgres.get(iPostgres++); | |
1562 | 937 checkPostgres(postgresChecker,lts,idPostgres); |
1391 | 938 } |
939 } finally { | |
940 close(searcher); | |
941 postgresChecker.close(); | |
942 } | |
943 } | |
944 | |
1562 | 945 private void checkPostgres(PostgresBackup.Checker postgresChecker,LuanToString lts,long id) |
1447 | 946 throws IOException, SQLException, LuanException, ParseException |
1430 | 947 { |
1454 | 948 //logger.info("check id "+id); |
1430 | 949 writeLock.lock(); |
950 try { | |
951 final IndexSearcher searcher = openSearcher(); | |
952 try { | |
953 LuanTable docPostgres = postgresChecker.getDoc(id); | |
954 TopDocs td = searcher.search(new TermQuery(term("id",id)),1); | |
955 LuanTable docLucene; | |
956 if( td.totalHits == 0 ) { | |
957 docLucene = null; | |
958 } else if( td.totalHits == 1 ) { | |
959 Document doc = searcher.doc( td.scoreDocs[0].doc ); | |
1562 | 960 docLucene = toTable(doc); |
1430 | 961 } else |
962 throw new RuntimeException(); | |
963 if( docPostgres == null ) { | |
964 if( docLucene != null ) | |
1454 | 965 logger.error("id "+id+" found in lucene but not postgres"); |
1430 | 966 return; |
967 } | |
968 if( docLucene == null ) { | |
1454 | 969 logger.error("id "+id+" found in postgres but not lucene"); |
1430 | 970 return; |
971 } | |
972 if( !equal(docPostgres,docLucene) ) { | |
1454 | 973 logger.error("id "+id+" not equal"); |
974 logger.error("lucene = "+lts.toString(docLucene)); | |
975 logger.error("postgres = "+lts.toString(docPostgres)); | |
1430 | 976 } |
977 } finally { | |
978 close(searcher); | |
979 } | |
980 } finally { | |
981 writeLock.unlock(); | |
982 } | |
983 } | |
984 | |
1453 | 985 private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException { |
986 return t1!=null && t2!=null && toJava(t1).equals(toJava(t2)); | |
1391 | 987 } |
988 | |
1453 | 989 private static Map toJava(LuanTable t) throws LuanException { |
990 Map map = t.asMap(); | |
1526 | 991 for( Iterator iter = map.entrySet().iterator(); iter.hasNext(); ) { |
992 Map.Entry entry = (Map.Entry)iter.next(); | |
1453 | 993 Object value = entry.getValue(); |
994 if( value instanceof LuanTable ) { | |
995 LuanTable v = (LuanTable)value; | |
996 if( !v.isList() ) | |
1454 | 997 logger.error("not list"); |
1530 | 998 List list = v.asList(); |
999 if( list.isEmpty() ) { | |
1526 | 1000 iter.remove(); |
1530 | 1001 } else if( list.size() == 1 ) { |
1002 entry.setValue(list.get(0)); | |
1526 | 1003 } else { |
1530 | 1004 entry.setValue(list); |
1526 | 1005 } |
1453 | 1006 } |
1007 } | |
1008 return map; | |
1009 } | |
230
4438cb2e04d0
start lucene
fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
parents:
diff
changeset
|
1010 } |