Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1345:6f8988830098

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
unique LuceneIndex per dir
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 25 Feb 2019 11:00:10 -0700
parents dc2af9d5463b
children efd1c6380f2c
comparison
equal deleted inserted replaced
1344:dc2af9d5463b 1345:6f8988830098
3 import java.io.Closeable; 3 import java.io.Closeable;
4 import java.io.File; 4 import java.io.File;
5 import java.io.FileOutputStream; 5 import java.io.FileOutputStream;
6 import java.io.FileInputStream; 6 import java.io.FileInputStream;
7 import java.io.IOException; 7 import java.io.IOException;
8 import java.util.Arrays;
8 import java.util.Iterator; 9 import java.util.Iterator;
9 import java.util.Map; 10 import java.util.Map;
11 import java.util.HashMap;
10 import java.util.List; 12 import java.util.List;
11 import java.util.ArrayList; 13 import java.util.ArrayList;
12 import java.util.Set; 14 import java.util.Set;
13 import java.util.HashSet; 15 import java.util.HashSet;
14 import java.util.Collections; 16 import java.util.Collections;
15 import java.util.concurrent.ConcurrentMap;
16 import java.util.concurrent.ConcurrentHashMap;
17 import java.util.concurrent.atomic.AtomicInteger; 17 import java.util.concurrent.atomic.AtomicInteger;
18 import java.util.concurrent.locks.Lock; 18 import java.util.concurrent.locks.Lock;
19 import java.util.concurrent.locks.ReentrantLock; 19 import java.util.concurrent.locks.ReentrantLock;
20 import java.util.zip.ZipOutputStream; 20 import java.util.zip.ZipOutputStream;
21 import java.util.zip.ZipEntry; 21 import java.util.zip.ZipEntry;
22 import org.apache.lucene.analysis.Analyzer; 22 import org.apache.lucene.analysis.Analyzer;
23 import org.apache.lucene.analysis.TokenStream; 23 import org.apache.lucene.analysis.TokenStream;
24 import org.apache.lucene.analysis.core.KeywordAnalyzer; 24 import org.apache.lucene.analysis.core.KeywordAnalyzer;
25 import org.apache.lucene.analysis.en.EnglishAnalyzer;
25 import org.apache.lucene.document.Document; 26 import org.apache.lucene.document.Document;
26 import org.apache.lucene.document.Field; 27 import org.apache.lucene.document.Field;
27 import org.apache.lucene.document.StoredField; 28 import org.apache.lucene.document.StoredField;
28 import org.apache.lucene.document.StringField; 29 import org.apache.lucene.document.StringField;
29 import org.apache.lucene.document.TextField; 30 import org.apache.lucene.document.TextField;
76 import luan.LuanRuntimeException; 77 import luan.LuanRuntimeException;
77 import luan.lib.logging.Logger; 78 import luan.lib.logging.Logger;
78 import luan.lib.logging.LoggerFactory; 79 import luan.lib.logging.LoggerFactory;
79 80
80 81
81 public final class LuceneIndex implements Closeable { 82 public final class LuceneIndex {
82 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); 83 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class);
83 84
85 private static final class Closer implements Closeable {
86 final LuceneIndex li;
87 boolean isClosed = false;
88 private final Exception created = new Exception("created");
89
90 Closer(Luan luan,LuceneIndex li) {
91 this.li = li;
92 luan.onClose(this);
93 }
94
95 public void close() throws IOException {
96 if( !isClosed ) {
97 li.close();
98 isClosed = true;
99 }
100 }
101
102 protected void finalize() throws Throwable {
103 if( !isClosed ) {
104 logger.error("not closed",created);
105 close();
106 }
107 super.finalize();
108 }
109 }
110
111 private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
112
113 public static Object[] getLuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields)
114 throws LuanException, IOException
115 {
116 String key = new File(indexDirStr).getCanonicalPath();
117 synchronized(indexes) {
118 LuceneIndex li = indexes.get(key);
119 if( li == null ) {
120 li = new LuceneIndex(indexDirStr,defaultFieldParser,defaultFields,key);
121 li.openCount = 1;
122 indexes.put(key,li);
123 } else {
124 if( defaultFieldParser != li.defaultFieldParser )
125 throw new LuanException("default_type doesn't match previous use");
126 if( !Arrays.equals(defaultFields,li.defaultFields) )
127 throw new LuanException("default_fields don't match previous use");
128 li.openCount++;
129 }
130 return new Object[]{li,new Closer(luan,li)};
131 }
132 }
133
134 private static final Version version = Version.LUCENE_4_9;
84 private static final String FLD_NEXT_ID = "nextId"; 135 private static final String FLD_NEXT_ID = "nextId";
85 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); 136 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
86 137 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
87 private static final Version version = Version.LUCENE_4_9; 138
88 private final ReentrantLock writeLock = new ReentrantLock(); 139 private final ReentrantLock writeLock = new ReentrantLock();
89 private final File indexDir; 140 private final File indexDir;
90 private SnapshotDeletionPolicy snapshotDeletionPolicy; 141 private SnapshotDeletionPolicy snapshotDeletionPolicy;
91 private IndexWriter writer; 142 private IndexWriter writer;
92 private DirectoryReader reader; 143 private DirectoryReader reader;
93 private IndexSearcher searcher; 144 private IndexSearcher searcher;
94 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); 145 private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>();
95 private boolean isClosed = true;
96 private final MultiFieldParser mfp; 146 private final MultiFieldParser mfp;
97 private final Analyzer analyzer; 147 private final Analyzer analyzer;
98 private final Exception created = new Exception("created"); 148
99
100 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
101 private File fileDir; 149 private File fileDir;
102 private int writeCount; 150 private int writeCount;
151 private AtomicInteger writeCounter = new AtomicInteger();
103 152
104 private Set<String> indexOnly = new HashSet<String>(); 153 private Set<String> indexOnly = new HashSet<String>();
105 154
106 public LuceneIndex(Luan luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) 155 private int openCount;
156 private final String key;
157 private final FieldParser defaultFieldParser;
158 private final String[] defaultFields;
159
160 private LuceneIndex(String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields,String key)
107 throws LuanException, IOException 161 throws LuanException, IOException
108 { 162 {
163 this.key = key;
164 this.defaultFieldParser = defaultFieldParser;
165 this.defaultFields = defaultFields;
109 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 166 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
110 mfp.fields.put( "type", STRING_FIELD_PARSER ); 167 mfp.fields.put( "type", STRING_FIELD_PARSER );
111 mfp.fields.put( "id", NumberFieldParser.LONG ); 168 mfp.fields.put( "id", NumberFieldParser.LONG );
112 File indexDir = new File(indexDirStr); 169 File indexDir = new File(indexDirStr);
113 this.indexDir = indexDir; 170 this.indexDir = indexDir;
115 if( defaultFieldParser instanceof StringFieldParser ) { 172 if( defaultFieldParser instanceof StringFieldParser ) {
116 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; 173 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
117 analyzer = sfp.analyzer; 174 analyzer = sfp.analyzer;
118 } 175 }
119 this.analyzer = analyzer; 176 this.analyzer = analyzer;
120 luan.onClose(this);
121 reopen(); 177 reopen();
122 } 178 }
123 179
124 public void reopen() throws LuanException, IOException { 180 public void reopen() throws LuanException, IOException {
125 if( !isClosed ) throw new RuntimeException();
126 isClosed = false;
127 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); 181 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
128 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); 182 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
129 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); 183 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
130 FSDirectory dir = FSDirectory.open(indexDir); 184 FSDirectory dir = FSDirectory.open(indexDir);
131 fileDir = dir.getDirectory(); 185 fileDir = dir.getDirectory();
132 globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger());
133 writer = new IndexWriter(dir,conf); 186 writer = new IndexWriter(dir,conf);
134 writer.commit(); // commit index creation 187 writer.commit(); // commit index creation
135 reader = DirectoryReader.open(dir); 188 reader = DirectoryReader.open(dir);
136 searcher = new IndexSearcher(reader); 189 searcher = new IndexSearcher(reader);
137 initId(); 190 initId();
138 } 191 }
139 192
140 private int globalWriteCount() {
141 return globalWriteCounters.get(fileDir).get();
142 }
143
144 private void wrote() { 193 private void wrote() {
145 globalWriteCounters.get(fileDir).incrementAndGet(); 194 writeCounter.incrementAndGet();
146 } 195 }
147 196
148 public void delete_all() throws IOException { 197 public void delete_all() throws IOException {
149 boolean commit = !writeLock.isHeldByCurrentThread(); 198 boolean commit = !writeLock.isHeldByCurrentThread();
150 writeLock.lock(); 199 writeLock.lock();
310 359
311 public String to_string() { 360 public String to_string() {
312 return writer.getDirectory().toString(); 361 return writer.getDirectory().toString();
313 } 362 }
314 363
315 public void close() throws IOException { 364 private synchronized void close() throws IOException {
316 if( !isClosed ) { 365 if( openCount > 0 ) {
317 writer.close(); 366 if( --openCount == 0 ) {
318 reader.close(); 367 doClose();
319 isClosed = true; 368 synchronized(indexes) {
320 } 369 indexes.remove(key);
321 } 370 }
322 371 }
323 protected void finalize() throws Throwable { 372 }
324 if( !isClosed ) { 373 }
325 logger.error("not closed",created); 374
326 close(); 375 public void doClose() throws IOException {
327 } 376 writer.close();
328 super.finalize(); 377 reader.close();
329 } 378 }
330
331 379
332 380
333 private static class DocFn extends LuanFunction { 381 private static class DocFn extends LuanFunction {
334 final IndexSearcher searcher; 382 final IndexSearcher searcher;
335 final Query query; 383 final Query query;
368 return true; 416 return true;
369 } 417 }
370 } 418 }
371 419
372 private synchronized IndexSearcher openSearcher() throws IOException { 420 private synchronized IndexSearcher openSearcher() throws IOException {
373 int gwc = globalWriteCount(); 421 int gwc = writeCounter.get();
374 if( writeCount != gwc ) { 422 if( writeCount != gwc ) {
375 writeCount = gwc; 423 writeCount = gwc;
376 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); 424 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
377 if( newReader != null ) { 425 if( newReader != null ) {
378 reader.decRef(); 426 reader.decRef();