Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1764:527c53b91a50

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lucene error handling
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 22 May 2023 20:43:52 -0600
parents 164c6ea53147
children c62324841dfb
comparison
equal deleted inserted replaced
1763:164c6ea53147 1764:527c53b91a50
1 package luan.modules.lucene; 1 package luan.modules.lucene;
2 2
3 import java.io.Closeable;
4 import java.io.File; 3 import java.io.File;
5 import java.io.FileOutputStream;
6 import java.io.FileInputStream;
7 import java.io.IOException; 4 import java.io.IOException;
8 import java.lang.ref.Reference; 5 import java.lang.ref.Reference;
9 import java.lang.ref.WeakReference; 6 import java.lang.ref.WeakReference;
10 import java.util.Arrays;
11 import java.util.Iterator;
12 import java.util.Map; 7 import java.util.Map;
13 import java.util.HashMap; 8 import java.util.HashMap;
14 import java.util.List;
15 import java.util.ArrayList; 9 import java.util.ArrayList;
16 import java.util.Set;
17 import java.util.HashSet;
18 import java.util.Collections;
19 import java.util.concurrent.atomic.AtomicInteger; 10 import java.util.concurrent.atomic.AtomicInteger;
20 import java.util.concurrent.locks.Lock;
21 import java.util.concurrent.locks.ReentrantLock; 11 import java.util.concurrent.locks.ReentrantLock;
22 import java.util.zip.ZipOutputStream;
23 import java.util.zip.ZipEntry;
24 import org.apache.lucene.analysis.Analyzer; 12 import org.apache.lucene.analysis.Analyzer;
25 import org.apache.lucene.analysis.TokenStream; 13 import org.apache.lucene.analysis.TokenStream;
26 import org.apache.lucene.analysis.core.KeywordAnalyzer; 14 import org.apache.lucene.analysis.core.KeywordAnalyzer;
27 import org.apache.lucene.analysis.core.SimpleAnalyzer; 15 import org.apache.lucene.analysis.core.SimpleAnalyzer;
28 import org.apache.lucene.analysis.en.EnglishAnalyzer; 16 import org.apache.lucene.analysis.en.EnglishAnalyzer;
29 import org.apache.lucene.document.Document; 17 import org.apache.lucene.document.Document;
30 import org.apache.lucene.document.Field;
31 import org.apache.lucene.document.StoredField;
32 import org.apache.lucene.document.StringField;
33 import org.apache.lucene.document.TextField;
34 import org.apache.lucene.document.IntField;
35 import org.apache.lucene.document.LongField;
36 import org.apache.lucene.document.DoubleField;
37 import org.apache.lucene.index.IndexableField;
38 import org.apache.lucene.index.IndexWriter;
39 import org.apache.lucene.index.IndexWriterConfig;
40 import org.apache.lucene.index.DirectoryReader; 18 import org.apache.lucene.index.DirectoryReader;
41 import org.apache.lucene.index.Term; 19 import org.apache.lucene.index.Term;
42 import org.apache.lucene.index.SnapshotDeletionPolicy; 20 import org.apache.lucene.index.SnapshotDeletionPolicy;
43 import org.apache.lucene.index.IndexCommit; 21 import org.apache.lucene.index.IndexCommit;
44 import org.apache.lucene.index.AtomicReaderContext; 22 import org.apache.lucene.index.AtomicReaderContext;
45 import org.apache.lucene.index.CheckIndex; 23 import org.apache.lucene.index.CheckIndex;
46 import org.apache.lucene.store.Directory;
47 import org.apache.lucene.store.FSDirectory; 24 import org.apache.lucene.store.FSDirectory;
48 import org.apache.lucene.util.Version; 25 import org.apache.lucene.util.Version;
49 import org.apache.lucene.util.BytesRef; 26 import org.apache.lucene.util.BytesRef;
50 import org.apache.lucene.util.NumericUtils; 27 import org.apache.lucene.util.NumericUtils;
51 import org.apache.lucene.search.Query; 28 import org.apache.lucene.search.Query;
52 import org.apache.lucene.search.PrefixQuery;
53 import org.apache.lucene.search.TermQuery;
54 import org.apache.lucene.search.MatchAllDocsQuery; 29 import org.apache.lucene.search.MatchAllDocsQuery;
55 import org.apache.lucene.search.TopDocs; 30 import org.apache.lucene.search.TopDocs;
56 import org.apache.lucene.search.Sort; 31 import org.apache.lucene.search.Sort;
57 import org.apache.lucene.search.SortField; 32 import org.apache.lucene.search.SortField;
58 import org.apache.lucene.search.IndexSearcher; 33 import org.apache.lucene.search.IndexSearcher;
62 import org.apache.lucene.search.Scorer; 37 import org.apache.lucene.search.Scorer;
63 import org.apache.lucene.search.Explanation; 38 import org.apache.lucene.search.Explanation;
64 import org.apache.lucene.search.highlight.Formatter; 39 import org.apache.lucene.search.highlight.Formatter;
65 import org.apache.lucene.search.highlight.Highlighter; 40 import org.apache.lucene.search.highlight.Highlighter;
66 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; 41 import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
67 import org.apache.lucene.search.highlight.Fragmenter;
68 import org.apache.lucene.search.highlight.NullFragmenter; 42 import org.apache.lucene.search.highlight.NullFragmenter;
69 import org.apache.lucene.search.highlight.SimpleSpanFragmenter; 43 import org.apache.lucene.search.highlight.SimpleSpanFragmenter;
70 import org.apache.lucene.search.highlight.QueryScorer; 44 import org.apache.lucene.search.highlight.QueryScorer;
71 import org.apache.lucene.search.highlight.TokenGroup; 45 import org.apache.lucene.search.highlight.TokenGroup;
72 import goodjava.lucene.analysis.LowercaseAnalyzer; 46 import goodjava.lucene.analysis.LowercaseAnalyzer;
87 import luan.Luan; 61 import luan.Luan;
88 import luan.LuanTable; 62 import luan.LuanTable;
89 import luan.LuanFunction; 63 import luan.LuanFunction;
90 import luan.LuanException; 64 import luan.LuanException;
91 import luan.LuanRuntimeException; 65 import luan.LuanRuntimeException;
92 import luan.modules.parsers.LuanToString;
93 import goodjava.logging.Logger; 66 import goodjava.logging.Logger;
94 import goodjava.logging.LoggerFactory; 67 import goodjava.logging.LoggerFactory;
95 68
96 69
97 public final class LuceneIndex { 70 public final class LuceneIndex {
188 161
189 public boolean reopen() throws IOException { 162 public boolean reopen() throws IOException {
190 fsDir = FSDirectory.open(indexDir); 163 fsDir = FSDirectory.open(indexDir);
191 boolean wasCreated = !fsDir.getDirectory().exists(); 164 boolean wasCreated = !fsDir.getDirectory().exists();
192 writer = new LuceneIndexWriter(fsDir,config); 165 writer = new LuceneIndexWriter(fsDir,config);
193 if( logDir != null ) { 166 try {
194 if( BackupIndexWriter.backupDomains == null ) { 167 if( logDir != null ) {
195 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime); 168 if( BackupIndexWriter.backupDomains == null ) {
196 } else { 169 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime);
197 writer = BackupIndexWriter.newWithRestore((LuceneIndexWriter)writer,logDir,logTime,domain,name); 170 } else {
198 } 171 writer = BackupIndexWriter.newWithRestore((LuceneIndexWriter)writer,logDir,logTime,domain,name);
199 } 172 }
200 reader = DirectoryReader.open(fsDir); 173 }
201 searcher = new IndexSearcher(reader); 174 reader = DirectoryReader.open(fsDir);
202 initId(); 175 searcher = new IndexSearcher(reader);
203 return wasCreated; 176 initId();
177 return wasCreated;
178 } catch(IOException e) {
179 writer.close();
180 throw e;
181 }
204 } 182 }
205 183
206 private void wrote() { 184 private void wrote() {
207 writeCounter.incrementAndGet(); 185 writeCounter.incrementAndGet();
208 } 186 }
356 default: 334 default:
357 throw new RuntimeException(); 335 throw new RuntimeException();
358 } 336 }
359 } 337 }
360 338
361 /*
362 public void backup(String zipFile) throws LuanException, IOException {
363 if( !zipFile.endsWith(".zip") )
364 throw new LuanException("file "+zipFile+" doesn't end with '.zip'");
365 IndexCommit ic = snapshotDeletionPolicy.snapshot();
366 try {
367 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
368 for( String fileName : ic.getFileNames() ) {
369 out.putNextEntry(new ZipEntry(fileName));
370 FileInputStream in = new FileInputStream(new File(indexDir,fileName));
371 Utils.copyAll(in,out);
372 in.close();
373 out.closeEntry();
374 }
375 out.close();
376 } finally {
377 snapshotDeletionPolicy.release(ic);
378 }
379 }
380 */
381 public SnapshotDeletionPolicy snapshotDeletionPolicy() { 339 public SnapshotDeletionPolicy snapshotDeletionPolicy() {
382 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy(); 340 return (SnapshotDeletionPolicy)writer.getLuceneIndexWriter().getConfig().getIndexDeletionPolicy();
383 } 341 }
384 342
385 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException { 343 public Object snapshot(Luan luan,LuanFunction fn) throws LuanException, IOException {