Mercurial Hosting > luan

comparison lucene/src/luan/modules/lucene/LuceneIndex.java @ 704:90c89138c234

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
optimize lucene
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 13 May 2016 17:13:13 -0600
parents b21d82ee5756
children 1ed9e55f0be8
comparison
equal deleted inserted replaced
703:6e6e9e73abaa 704:90c89138c234
10 import java.util.List; 10 import java.util.List;
11 import java.util.ArrayList; 11 import java.util.ArrayList;
12 import java.util.Set; 12 import java.util.Set;
13 import java.util.HashSet; 13 import java.util.HashSet;
14 import java.util.Collections; 14 import java.util.Collections;
15 import java.util.concurrent.ConcurrentMap;
16 import java.util.concurrent.ConcurrentHashMap;
17 import java.util.concurrent.atomic.AtomicInteger;
15 import java.util.concurrent.locks.Lock; 18 import java.util.concurrent.locks.Lock;
16 import java.util.concurrent.locks.ReentrantLock; 19 import java.util.concurrent.locks.ReentrantLock;
17 import java.util.zip.ZipOutputStream; 20 import java.util.zip.ZipOutputStream;
18 import java.util.zip.ZipEntry; 21 import java.util.zip.ZipEntry;
19 import org.apache.lucene.analysis.Analyzer; 22 import org.apache.lucene.analysis.Analyzer;
90 private boolean isClosed = false; 93 private boolean isClosed = false;
91 private final MultiFieldParser mfp; 94 private final MultiFieldParser mfp;
92 public final LuanTable indexed_only_fields = new LuanTable(); 95 public final LuanTable indexed_only_fields = new LuanTable();
93 private final Analyzer analyzer; 96 private final Analyzer analyzer;
94 97
98 private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>();
99 private final File fileDir;
100 private int writeCount;
101
95 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) throws LuanException, IOException { 102 public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) throws LuanException, IOException {
96 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 103 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
97 mfp.fields.put( "type", STRING_FIELD_PARSER ); 104 mfp.fields.put( "type", STRING_FIELD_PARSER );
98 mfp.fields.put( "id", NumberFieldParser.LONG ); 105 mfp.fields.put( "id", NumberFieldParser.LONG );
99 File indexDir = new File(indexDirStr); 106 File indexDir = new File(indexDirStr);
100 this.indexDir = indexDir; 107 this.indexDir = indexDir;
101 Directory dir = FSDirectory.open(indexDir); 108 FSDirectory dir = FSDirectory.open(indexDir);
109 fileDir = dir.getDirectory();
110 globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger());
102 Version version = Version.LUCENE_4_9; 111 Version version = Version.LUCENE_4_9;
103 Analyzer analyzer = STRING_FIELD_PARSER.analyzer; 112 Analyzer analyzer = STRING_FIELD_PARSER.analyzer;
104 if( defaultFieldParser instanceof StringFieldParser ) { 113 if( defaultFieldParser instanceof StringFieldParser ) {
105 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; 114 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
106 analyzer = sfp.analyzer; 115 analyzer = sfp.analyzer;
115 luan.onClose(this); 124 luan.onClose(this);
116 searcher = new IndexSearcher(reader); 125 searcher = new IndexSearcher(reader);
117 initId(); 126 initId();
118 } 127 }
119 128
120 129 private int globalWriteCount() {
130 return globalWriteCounters.get(fileDir).get();
131 }
132
133 private void wrote() {
134 globalWriteCounters.get(fileDir).incrementAndGet();
135 }
121 136
122 public void delete_all() throws IOException { 137 public void delete_all() throws IOException {
123 boolean commit = !writeLock.isHeldByCurrentThread(); 138 boolean commit = !writeLock.isHeldByCurrentThread();
124 writeLock.lock(); 139 writeLock.lock();
125 try { 140 try {
126 writer.deleteAll(); 141 writer.deleteAll();
127 id = idLim = 0; 142 id = idLim = 0;
128 if(commit) writer.commit(); 143 if(commit) writer.commit();
129 } finally { 144 } finally {
145 wrote();
130 writeLock.unlock(); 146 writeLock.unlock();
131 } 147 }
132 } 148 }
133 149
134 private static Term term(String key,long value) { 150 private static Term term(String key,long value) {
144 writeLock.lock(); 160 writeLock.lock();
145 try { 161 try {
146 writer.deleteDocuments(query); 162 writer.deleteDocuments(query);
147 if(commit) writer.commit(); 163 if(commit) writer.commit();
148 } finally { 164 } finally {
165 wrote();
149 writeLock.unlock(); 166 writeLock.unlock();
150 } 167 }
151 } 168 }
152 169
153 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException { 170 public void save(LuanState luan,LuanTable doc) throws LuanException, IOException {
195 } else { 212 } else {
196 writer.updateDocument( term("id",id), toLucene(luan,doc,indexedOnlySet) ); 213 writer.updateDocument( term("id",id), toLucene(luan,doc,indexedOnlySet) );
197 } 214 }
198 if(commit) writer.commit(); 215 if(commit) writer.commit();
199 } finally { 216 } finally {
217 wrote();
200 writeLock.unlock(); 218 writeLock.unlock();
201 } 219 }
202 } 220 }
203 221
204 public void update_in_transaction(LuanState luan,LuanFunction fn) throws IOException, LuanException { 222 public void update_in_transaction(LuanState luan,LuanFunction fn) throws IOException, LuanException {
206 writeLock.lock(); 224 writeLock.lock();
207 try { 225 try {
208 fn.call(luan); 226 fn.call(luan);
209 if(commit) writer.commit(); 227 if(commit) writer.commit();
210 } finally { 228 } finally {
229 wrote();
211 writeLock.unlock(); 230 writeLock.unlock();
212 } 231 }
213 } 232 }
214 233
215 234
250 idLim += idBatch; 269 idLim += idBatch;
251 LuanTable doc = new LuanTable(); 270 LuanTable doc = new LuanTable();
252 doc.rawPut( "type", "next_id" ); 271 doc.rawPut( "type", "next_id" );
253 doc.rawPut( FLD_NEXT_ID, idLim ); 272 doc.rawPut( FLD_NEXT_ID, idLim );
254 writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc,Collections.EMPTY_SET)); 273 writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc,Collections.EMPTY_SET));
274 wrote();
255 } 275 }
256 return id; 276 return id;
257 } 277 }
258 278
259 279
329 return true; 349 return true;
330 } 350 }
331 } 351 }
332 352
333 private synchronized IndexSearcher openSearcher() throws IOException { 353 private synchronized IndexSearcher openSearcher() throws IOException {
334 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); 354 int gwc = globalWriteCount();
335 if( newReader != null ) { 355 if( writeCount != gwc ) {
336 reader.decRef(); 356 writeCount = gwc;
337 reader = newReader; 357 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
338 searcher = new IndexSearcher(reader); 358 if( newReader != null ) {
359 reader.decRef();
360 reader = newReader;
361 searcher = new IndexSearcher(reader);
362 }
339 } 363 }
340 reader.incRef(); 364 reader.incRef();
341 return searcher; 365 return searcher;
342 } 366 }
343 367