Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1397:0dc9837c16be

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix lucene issues
author Franklin Schmidt <fschmidt@gmail.com>
date Wed, 11 Sep 2019 01:31:21 -0600
parents 9dfff82dfc59
children 67c0e47b5be3
comparison
equal deleted inserted replaced
1396:a5f61890ad84 1397:0dc9837c16be
3 import java.io.Closeable; 3 import java.io.Closeable;
4 import java.io.File; 4 import java.io.File;
5 import java.io.FileOutputStream; 5 import java.io.FileOutputStream;
6 import java.io.FileInputStream; 6 import java.io.FileInputStream;
7 import java.io.IOException; 7 import java.io.IOException;
8 import java.lang.ref.Reference;
9 import java.lang.ref.WeakReference;
8 import java.sql.SQLException; 10 import java.sql.SQLException;
9 import java.util.Arrays; 11 import java.util.Arrays;
10 import java.util.Iterator; 12 import java.util.Iterator;
11 import java.util.Map; 13 import java.util.Map;
12 import java.util.HashMap; 14 import java.util.HashMap;
81 import luan.modules.parsers.LuanToString; 83 import luan.modules.parsers.LuanToString;
82 import luan.lib.logging.Logger; 84 import luan.lib.logging.Logger;
83 import luan.lib.logging.LoggerFactory; 85 import luan.lib.logging.LoggerFactory;
84 86
85 87
86 public final class LuceneIndex { 88 public final class LuceneIndex implements Closeable {
87 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class); 89 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
88 90
89 private static final class Closer implements Closeable { 91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
90 final LuceneIndex li; 92
91 boolean isClosed = false; 93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
92 private final Exception created = new Exception("created");
93
94 Closer(Luan luan,LuceneIndex li) {
95 this.li = li;
96 luan.onClose(this);
97 }
98
99 public void close() throws IOException {
100 if( !isClosed ) {
101 try {
102 li.close();
103 } catch(SQLException e) {
104 throw new RuntimeException(e);
105 }
106 isClosed = true;
107 }
108 }
109
110 protected void finalize() throws Throwable {
111 if( !isClosed ) {
112 sysLogger.error("not closed",created);
113 close();
114 }
115 super.finalize();
116 }
117 }
118
119 private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
120
121 public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,LuanFunction completer,LuanTable postgresSpec)
122 throws LuanException, IOException, ClassNotFoundException, SQLException 94 throws LuanException, IOException, ClassNotFoundException, SQLException
123 { 95 {
124 String key = indexDir.getCanonicalPath(); 96 String key = indexDir.getCanonicalPath();
125 synchronized(indexes) { 97 synchronized(indexes) {
126 LuceneIndex li = indexes.get(key); 98 Reference<LuceneIndex> ref = indexes.get(key);
127 if( li == null ) { 99 if( ref != null ) {
128 li = new LuceneIndex(luan,indexDir,defaultFieldParser,defaultFields,key,completer,postgresSpec); 100 LuceneIndex li = ref.get();
129 li.openCount = 1; 101 if( li != null )
130 indexes.put(key,li); 102 li.closeWriter();
131 } else { 103 }
132 if( defaultFieldParser != li.defaultFieldParser ) 104 LuceneIndex li = new LuceneIndex(luan,indexDir,options);
133 throw new LuanException("default_type doesn't match previous use"); 105 indexes.put(key, new WeakReference<LuceneIndex>(li));
134 if( !Arrays.equals(defaultFields,li.defaultFields) ) 106 return li;
135 throw new LuanException("default_fields don't match previous use");
136 li.openCount++;
137 }
138 return new Object[]{li,new Closer(luan,li)};
139 } 107 }
140 } 108 }
141 109
142 private static final Version version = Version.LUCENE_4_9; 110 private static final Version version = Version.LUCENE_4_9;
143 private static final String FLD_NEXT_ID = "nextId"; 111 private static final String FLD_NEXT_ID = "nextId";
160 private int writeCount; 128 private int writeCount;
161 private AtomicInteger writeCounter = new AtomicInteger(); 129 private AtomicInteger writeCounter = new AtomicInteger();
162 130
163 private Set<String> indexOnly = new HashSet<String>(); 131 private Set<String> indexOnly = new HashSet<String>();
164 132
165 private int openCount; 133 private boolean isClosed = false;
166 private final String key; 134 private final Exception created = new Exception("created");
167 private final FieldParser defaultFieldParser; 135 private final FieldParser defaultFieldParser;
168 private final String[] defaultFields; 136 private final String[] defaultFields;
169 137
170 private final PostgresBackup postgresBackup; 138 private final PostgresBackup postgresBackup;
171 139 private boolean wasCreated;
172 private LuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanFunction completer,LuanTable postgresSpec) 140
141 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
173 throws LuanException, IOException, ClassNotFoundException, SQLException 142 throws LuanException, IOException, ClassNotFoundException, SQLException
174 { 143 {
144 Map map = options.asMap();
145 FieldParser defaultFieldParser = (FieldParser)map.remove("default_type");
146 LuanTable defaultFieldsTbl = Utils.removeTable(map,"default_fields");
147 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
148 LuanFunction completer = Utils.removeFunction(map,"completer");
149 LuanTable postgresSpec = Utils.removeTable(map,"postgres_spec");
150 Utils.checkEmpty(map);
151
175 this.luanLogger = luan.getLogger(LuceneIndex.class); 152 this.luanLogger = luan.getLogger(LuceneIndex.class);
176 this.key = key;
177 this.defaultFieldParser = defaultFieldParser; 153 this.defaultFieldParser = defaultFieldParser;
178 this.defaultFields = defaultFields; 154 this.defaultFields = defaultFields;
179 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 155 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
180 mfp.fields.put( "type", STRING_FIELD_PARSER ); 156 mfp.fields.put( "type", STRING_FIELD_PARSER );
181 mfp.fields.put( "id", NumberFieldParser.LONG ); 157 mfp.fields.put( "id", NumberFieldParser.LONG );
184 if( defaultFieldParser instanceof StringFieldParser ) { 160 if( defaultFieldParser instanceof StringFieldParser ) {
185 StringFieldParser sfp = (StringFieldParser)defaultFieldParser; 161 StringFieldParser sfp = (StringFieldParser)defaultFieldParser;
186 analyzer = sfp.analyzer; 162 analyzer = sfp.analyzer;
187 } 163 }
188 this.analyzer = analyzer; 164 this.analyzer = analyzer;
189 boolean wasCreated = reopen(); 165 wasCreated = reopen();
190 if( postgresSpec == null ) { 166 if( postgresSpec == null ) {
191 postgresBackup = null; 167 postgresBackup = null;
192 } else { 168 } else {
193 if( completer == null ) 169 if( completer == null )
194 throw new LuanException("completer is required for postgres_spec"); 170 throw new LuanException("completer is required for postgres_spec");
195 Map spec = postgresSpec.asMap(); 171 Map spec = postgresSpec.asMap();
196 postgresBackup = new PostgresBackup(luan,spec); 172 postgresBackup = new PostgresBackup(luan,spec);
197 if( postgresBackup != null ) { 173 if( !wasCreated && postgresBackup.wasCreated ) {
198 if( !wasCreated && postgresBackup.wasCreated ) { 174 luanLogger.error("rebuilding postgres backup");
199 luanLogger.error("rebuilding postgres backup"); 175 rebuild_postgres_backup(completer);
200 rebuild_postgres_backup(completer); 176 /*
201 } else if( wasCreated && !postgresBackup.wasCreated ) { 177 } else if( wasCreated && !postgresBackup.wasCreated ) {
202 luanLogger.error("restoring from postgres"); 178 luanLogger.error("restoring from postgres");
203 restore_from_postgres(); 179 restore_from_postgres();
204 } 180 */
205 } 181 }
206 } 182 }
183 luan.onClose(this);
184 }
185
186 protected void finalize() throws Throwable {
187 if( !isClosed ) {
188 sysLogger.error("not closed",created);
189 close();
190 }
191 super.finalize();
207 } 192 }
208 193
209 public boolean reopen() throws IOException { 194 public boolean reopen() throws IOException {
210 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); 195 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
211 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); 196 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
447 432
448 public String to_string() { 433 public String to_string() {
449 return writer.getDirectory().toString(); 434 return writer.getDirectory().toString();
450 } 435 }
451 436
452 private synchronized void close() throws IOException, SQLException { 437 public synchronized void close() throws IOException {
453 if( openCount > 0 ) { 438 try {
454 if( --openCount == 0 ) { 439 doClose();
455 doClose(); 440 } catch(SQLException e) {
456 synchronized(indexes) { 441 throw new RuntimeException(e);
457 indexes.remove(key); 442 }
458 } 443 isClosed = true;
459 }
460 }
461 } 444 }
462 445
463 public void doClose() throws IOException, SQLException { 446 public void doClose() throws IOException, SQLException {
464 writer.close(); 447 writer.close();
465 reader.close(); 448 reader.close();
466 if( postgresBackup != null ) 449 if( postgresBackup != null )
467 postgresBackup.close(); 450 postgresBackup.close();
451 }
452
453 private void closeWriter() throws IOException {
454 writeLock.lock();
455 try {
456 writer.close();
457 } finally {
458 writeLock.unlock();
459 }
468 } 460 }
469 461
470 462
471 private static class DocFn extends LuanFunction { 463 private static class DocFn extends LuanFunction {
472 final IndexSearcher searcher; 464 final IndexSearcher searcher;
863 } 855 }
864 856
865 public void restore_from_postgres() 857 public void restore_from_postgres()
866 throws IOException, LuanException, SQLException 858 throws IOException, LuanException, SQLException
867 { 859 {
860 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
861 luanLogger.error("restoring from postgres");
862 force_restore_from_postgres();
863 wasCreated = false;
864 }
865 }
866
867 public void force_restore_from_postgres()
868 throws IOException, LuanException, SQLException
869 {
868 luanLogger.warn("start restore_from_postgres"); 870 luanLogger.warn("start restore_from_postgres");
869 if( postgresBackup==null ) 871 if( postgresBackup==null )
870 throw new NullPointerException(); 872 throw new NullPointerException();
871 if( writeLock.isHeldByCurrentThread() ) 873 if( writeLock.isHeldByCurrentThread() )
872 throw new RuntimeException(); 874 throw new RuntimeException();