Mercurial Hosting > luan
comparison src/luan/modules/lucene/LuceneIndex.java @ 1538:634f6765830e
use goodjava/lucene/logging
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Fri, 07 Aug 2020 21:42:16 -0600 |
parents | e73b72a510b4 |
children | c27dc6af87ca |
comparison
equal
deleted
inserted
replaced
1537:f7649ad6e3e7 | 1538:634f6765830e |
---|---|
76 import goodjava.lucene.queryparser.NumberFieldParser; | 76 import goodjava.lucene.queryparser.NumberFieldParser; |
77 import goodjava.lucene.api.GoodIndexWriter; | 77 import goodjava.lucene.api.GoodIndexWriter; |
78 import goodjava.lucene.api.LuceneIndexWriter; | 78 import goodjava.lucene.api.LuceneIndexWriter; |
79 import goodjava.lucene.api.GoodIndexWriterConfig; | 79 import goodjava.lucene.api.GoodIndexWriterConfig; |
80 import goodjava.lucene.api.LuceneUtils; | 80 import goodjava.lucene.api.LuceneUtils; |
81 import goodjava.lucene.logging.LoggingIndexWriter; | |
81 import goodjava.parser.ParseException; | 82 import goodjava.parser.ParseException; |
82 import luan.modules.Utils; | 83 import luan.modules.Utils; |
83 import luan.Luan; | 84 import luan.Luan; |
84 import luan.LuanTable; | 85 import luan.LuanTable; |
85 import luan.LuanFunction; | 86 import luan.LuanFunction; |
119 private static final Version luceneVersion = Version.LUCENE_4_9; | 120 private static final Version luceneVersion = Version.LUCENE_4_9; |
120 private static final String FLD_NEXT_ID = "nextId"; | 121 private static final String FLD_NEXT_ID = "nextId"; |
121 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); | 122 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); |
122 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion)); | 123 public static final StringFieldParser LOWERCASE_FIELD_PARSER = new StringFieldParser(new LowercaseAnalyzer(luceneVersion)); |
123 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion)); | 124 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(luceneVersion)); |
125 private static final SortField ID_SORT = new SortField("id",SortField.Type.LONG); | |
124 | 126 |
125 private final Object version; | 127 private final Object version; |
126 | 128 |
127 private final ReentrantLock writeLock = new ReentrantLock(); | 129 private final ReentrantLock writeLock = new ReentrantLock(); |
128 private final File indexDir; | 130 private final File indexDir; |
138 private AtomicInteger writeCounter = new AtomicInteger(); | 140 private AtomicInteger writeCounter = new AtomicInteger(); |
139 private final GoodIndexWriterConfig config; | 141 private final GoodIndexWriterConfig config; |
140 | 142 |
141 private final PostgresBackup postgresBackup; | 143 private final PostgresBackup postgresBackup; |
142 private boolean wasCreated; | 144 private boolean wasCreated; |
145 private final File logDir; | |
143 | 146 |
144 private LuceneIndex(Luan luan,File indexDir,LuanTable options) | 147 private LuceneIndex(Luan luan,File indexDir,LuanTable options) |
145 throws LuanException, IOException, ClassNotFoundException, SQLException | 148 throws LuanException, IOException, ClassNotFoundException, SQLException |
146 { | 149 { |
147 options = new LuanTable(options); | 150 options = new LuanTable(options); |
149 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type"); | 152 FieldParser defaultFieldParser = (FieldParser)options.remove("default_type"); |
150 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); | 153 LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); |
151 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); | 154 String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); |
152 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); | 155 LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); |
153 LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); | 156 LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); |
157 logDir = (File)options.remove("log_dir"); | |
154 Utils.checkEmpty(options); | 158 Utils.checkEmpty(options); |
155 | 159 |
156 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); | 160 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); |
157 mfp.fields.put( "type", STRING_FIELD_PARSER ); | 161 mfp.fields.put( "type", STRING_FIELD_PARSER ); |
158 mfp.fields.put( "id", NumberFieldParser.LONG ); | 162 mfp.fields.put( "id", NumberFieldParser.LONG ); |
183 | 187 |
184 public boolean reopen() throws IOException { | 188 public boolean reopen() throws IOException { |
185 fsDir = FSDirectory.open(indexDir); | 189 fsDir = FSDirectory.open(indexDir); |
186 boolean wasCreated = !fsDir.getDirectory().exists(); | 190 boolean wasCreated = !fsDir.getDirectory().exists(); |
187 writer = new LuceneIndexWriter(fsDir,config); | 191 writer = new LuceneIndexWriter(fsDir,config); |
192 if( logDir != null ) | |
193 writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir); | |
188 reader = DirectoryReader.open(fsDir); | 194 reader = DirectoryReader.open(fsDir); |
189 searcher = new IndexSearcher(reader); | 195 searcher = new IndexSearcher(reader); |
190 initId(); | 196 initId(); |
191 return wasCreated; | 197 return wasCreated; |
192 } | 198 } |
779 msg += " with postgres"; | 785 msg += " with postgres"; |
780 logger.info(msg); | 786 logger.info(msg); |
781 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); | 787 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); |
782 if( !status.clean ) | 788 if( !status.clean ) |
783 logger.error("index not clean"); | 789 logger.error("index not clean"); |
784 if( hasPostgres ) | 790 if( writer instanceof LoggingIndexWriter ) { |
791 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; | |
792 logger.info("log check"); | |
793 boolean ok = loggingWriter.check(ID_SORT); | |
794 } | |
795 if( hasPostgres ) { | |
796 logger.info("postgres check"); | |
785 checkPostgres(luan); | 797 checkPostgres(luan); |
798 } | |
786 logger.info("end check"); | 799 logger.info("end check"); |
800 } | |
801 | |
802 public void rebuild_log() throws IOException { | |
803 logger.info("start rebuild_log"); | |
804 LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; | |
805 loggingWriter.newLogs(); | |
806 logger.info("end rebuild_log"); | |
787 } | 807 } |
788 | 808 |
789 private void checkPostgres(Luan luan) | 809 private void checkPostgres(Luan luan) |
790 throws IOException, SQLException, LuanException, ParseException | 810 throws IOException, SQLException, LuanException, ParseException |
791 { | 811 { |
792 //logger.info("start postgres check"); | |
793 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); | 812 final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); |
794 final IndexSearcher searcher = openSearcher(); | 813 final IndexSearcher searcher = openSearcher(); |
795 try { | 814 try { |
796 final List<Long> idsLucene = new ArrayList<Long>(); | 815 final List<Long> idsLucene = new ArrayList<Long>(); |
797 Query query = new PrefixQuery(new Term("id")); | 816 Query query = new PrefixQuery(new Term("id")); |