Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1398:67c0e47b5be3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
more lucene
author Franklin Schmidt <fschmidt@gmail.com>
date Wed, 11 Sep 2019 15:48:49 -0600
parents 0dc9837c16be
children 38a1c1b4279a
comparison
equal deleted inserted replaced
1397:0dc9837c16be 1398:67c0e47b5be3
83 import luan.modules.parsers.LuanToString; 83 import luan.modules.parsers.LuanToString;
84 import luan.lib.logging.Logger; 84 import luan.lib.logging.Logger;
85 import luan.lib.logging.LoggerFactory; 85 import luan.lib.logging.LoggerFactory;
86 86
87 87
88 public final class LuceneIndex implements Closeable { 88 public final class LuceneIndex {
89 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class); 89 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
90 90
91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>(); 91 private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
92 92
93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options) 93 public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
96 String key = indexDir.getCanonicalPath(); 96 String key = indexDir.getCanonicalPath();
97 synchronized(indexes) { 97 synchronized(indexes) {
98 Reference<LuceneIndex> ref = indexes.get(key); 98 Reference<LuceneIndex> ref = indexes.get(key);
99 if( ref != null ) { 99 if( ref != null ) {
100 LuceneIndex li = ref.get(); 100 LuceneIndex li = ref.get();
101 if( li != null ) 101 if( li != null ) {
102 Object version = options.get("version");
103 if( version==null || version.equals(li.version) )
104 return li;
102 li.closeWriter(); 105 li.closeWriter();
106 }
103 } 107 }
104 LuceneIndex li = new LuceneIndex(luan,indexDir,options); 108 LuceneIndex li = new LuceneIndex(luan,indexDir,options);
105 indexes.put(key, new WeakReference<LuceneIndex>(li)); 109 indexes.put(key, new WeakReference<LuceneIndex>(li));
106 return li; 110 return li;
107 } 111 }
108 } 112 }
109 113
110 private static final Version version = Version.LUCENE_4_9; 114 private static final Version luceneVersion = Version.LUCENE_4_9;
111 private static final String FLD_NEXT_ID = "nextId"; 115 private static final String FLD_NEXT_ID = "nextId";
112 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); 116 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
113 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version)); 117 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion));
114 118
115 private final Logger luanLogger; 119 private final Logger luanLogger;
120 private final Object version;
116 121
117 private final ReentrantLock writeLock = new ReentrantLock(); 122 private final ReentrantLock writeLock = new ReentrantLock();
118 private final File indexDir; 123 private final File indexDir;
119 private SnapshotDeletionPolicy snapshotDeletionPolicy; 124 private SnapshotDeletionPolicy snapshotDeletionPolicy;
120 private IndexWriter writer; 125 private IndexWriter writer;
127 private FSDirectory fsDir; 132 private FSDirectory fsDir;
128 private int writeCount; 133 private int writeCount;
129 private AtomicInteger writeCounter = new AtomicInteger(); 134 private AtomicInteger writeCounter = new AtomicInteger();
130 135
131 private Set<String> indexOnly = new HashSet<String>(); 136 private Set<String> indexOnly = new HashSet<String>();
132
133 private boolean isClosed = false;
134 private final Exception created = new Exception("created");
135 private final FieldParser defaultFieldParser; 137 private final FieldParser defaultFieldParser;
136 private final String[] defaultFields; 138 private final String[] defaultFields;
137 139
138 private final PostgresBackup postgresBackup; 140 private final PostgresBackup postgresBackup;
139 private boolean wasCreated; 141 private boolean wasCreated;
140 142
141 private LuceneIndex(Luan luan,File indexDir,LuanTable options) 143 private LuceneIndex(Luan luan,File indexDir,LuanTable options)
142 throws LuanException, IOException, ClassNotFoundException, SQLException 144 throws LuanException, IOException, ClassNotFoundException, SQLException
143 { 145 {
144 Map map = options.asMap(); 146 Map map = options.asMap();
147 this.version = map.remove("version");
145 FieldParser defaultFieldParser = (FieldParser)map.remove("default_type"); 148 FieldParser defaultFieldParser = (FieldParser)map.remove("default_type");
146 LuanTable defaultFieldsTbl = Utils.removeTable(map,"default_fields"); 149 LuanTable defaultFieldsTbl = Utils.removeTable(map,"default_fields");
147 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); 150 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
148 LuanFunction completer = Utils.removeFunction(map,"completer"); 151 LuanFunction completer = Utils.removeFunction(map,"completer");
149 LuanTable postgresSpec = Utils.removeTable(map,"postgres_spec"); 152 LuanTable postgresSpec = Utils.removeTable(map,"postgres_spec");
178 luanLogger.error("restoring from postgres"); 181 luanLogger.error("restoring from postgres");
179 restore_from_postgres(); 182 restore_from_postgres();
180 */ 183 */
181 } 184 }
182 } 185 }
183 luan.onClose(this);
184 }
185
186 protected void finalize() throws Throwable {
187 if( !isClosed ) {
188 sysLogger.error("not closed",created);
189 close();
190 }
191 super.finalize();
192 } 186 }
193 187
194 public boolean reopen() throws IOException { 188 public boolean reopen() throws IOException {
195 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); 189 IndexWriterConfig conf = new IndexWriterConfig(luceneVersion,analyzer);
196 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); 190 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
197 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); 191 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
198 fsDir = FSDirectory.open(indexDir); 192 fsDir = FSDirectory.open(indexDir);
199 boolean wasCreated = !fsDir.getDirectory().exists(); 193 boolean wasCreated = !fsDir.getDirectory().exists();
200 writer = new IndexWriter(fsDir,conf); 194 writer = new IndexWriter(fsDir,conf);
432 426
433 public String to_string() { 427 public String to_string() {
434 return writer.getDirectory().toString(); 428 return writer.getDirectory().toString();
435 } 429 }
436 430
437 public synchronized void close() throws IOException { 431 protected void finalize() throws Throwable {
438 try { 432 close();
439 doClose(); 433 super.finalize();
440 } catch(SQLException e) { 434 }
441 throw new RuntimeException(e); 435
442 } 436 public void close() throws IOException, SQLException {
443 isClosed = true; 437 closeWriter();
444 }
445
446 public void doClose() throws IOException, SQLException {
447 writer.close();
448 reader.close(); 438 reader.close();
449 if( postgresBackup != null ) 439 }
450 postgresBackup.close(); 440
451 } 441 private void closeWriter() throws IOException, SQLException {
452
453 private void closeWriter() throws IOException {
454 writeLock.lock(); 442 writeLock.lock();
455 try { 443 try {
456 writer.close(); 444 writer.close();
445 if( postgresBackup != null )
446 postgresBackup.close();
457 } finally { 447 } finally {
458 writeLock.unlock(); 448 writeLock.unlock();
459 } 449 }
460 } 450 }
461 451
858 throws IOException, LuanException, SQLException 848 throws IOException, LuanException, SQLException
859 { 849 {
860 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) { 850 if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
861 luanLogger.error("restoring from postgres"); 851 luanLogger.error("restoring from postgres");
862 force_restore_from_postgres(); 852 force_restore_from_postgres();
863 wasCreated = false;
864 } 853 }
865 } 854 }
866 855
867 public void force_restore_from_postgres() 856 public void force_restore_from_postgres()
868 throws IOException, LuanException, SQLException 857 throws IOException, LuanException, SQLException
880 postgresBackup.restoreLucene(this); 869 postgresBackup.restoreLucene(this);
881 id = idLim = nextId; 870 id = idLim = nextId;
882 saveNextId(nextId); 871 saveNextId(nextId);
883 ok = true; 872 ok = true;
884 writer.commit(); 873 writer.commit();
874 wasCreated = false;
885 } finally { 875 } finally {
886 if( !ok ) { 876 if( !ok ) {
887 writer.rollback(); 877 writer.rollback();
888 reopen(); 878 reopen();
889 } 879 }