Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1393:cc0dbca576dc

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
better logging
author Franklin Schmidt <fschmidt@gmail.com>
date Fri, 06 Sep 2019 05:09:56 -0600
parents 002152af497a
children 9dfff82dfc59
comparison
equal deleted inserted replaced
1392:002152af497a 1393:cc0dbca576dc
77 import luan.LuanTable; 77 import luan.LuanTable;
78 import luan.LuanFunction; 78 import luan.LuanFunction;
79 import luan.LuanException; 79 import luan.LuanException;
80 import luan.LuanRuntimeException; 80 import luan.LuanRuntimeException;
81 import luan.modules.parsers.LuanToString; 81 import luan.modules.parsers.LuanToString;
82 import luan.modules.logging.LuanLogger;
83 import luan.lib.logging.Logger; 82 import luan.lib.logging.Logger;
84 import luan.lib.logging.LoggerFactory; 83 import luan.lib.logging.LoggerFactory;
85 84
86 85
87 public final class LuceneIndex { 86 public final class LuceneIndex {
88 private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); 87 private static final Logger sysLogger = LoggerFactory.getLogger(LuceneIndex.class);
89 88
90 private static final class Closer implements Closeable { 89 private static final class Closer implements Closeable {
91 final LuceneIndex li; 90 final LuceneIndex li;
92 boolean isClosed = false; 91 boolean isClosed = false;
93 private final Exception created = new Exception("created"); 92 private final Exception created = new Exception("created");
108 } 107 }
109 } 108 }
110 109
111 protected void finalize() throws Throwable { 110 protected void finalize() throws Throwable {
112 if( !isClosed ) { 111 if( !isClosed ) {
113 logger.error("not closed",created); 112 sysLogger.error("not closed",created);
114 close(); 113 close();
115 } 114 }
116 super.finalize(); 115 super.finalize();
117 } 116 }
118 } 117 }
143 private static final Version version = Version.LUCENE_4_9; 142 private static final Version version = Version.LUCENE_4_9;
144 private static final String FLD_NEXT_ID = "nextId"; 143 private static final String FLD_NEXT_ID = "nextId";
145 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); 144 public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer());
146 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version)); 145 public static final StringFieldParser ENGLISH_FIELD_PARSER = new StringFieldParser(new EnglishAnalyzer(version));
147 146
147 private final Logger luanLogger;
148
148 private final ReentrantLock writeLock = new ReentrantLock(); 149 private final ReentrantLock writeLock = new ReentrantLock();
149 private final File indexDir; 150 private final File indexDir;
150 private SnapshotDeletionPolicy snapshotDeletionPolicy; 151 private SnapshotDeletionPolicy snapshotDeletionPolicy;
151 private IndexWriter writer; 152 private IndexWriter writer;
152 private DirectoryReader reader; 153 private DirectoryReader reader;
169 private final PostgresBackup postgresBackup; 170 private final PostgresBackup postgresBackup;
170 171
171 private LuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanTable postgresSpec) 172 private LuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanTable postgresSpec)
172 throws LuanException, IOException, ClassNotFoundException, SQLException 173 throws LuanException, IOException, ClassNotFoundException, SQLException
173 { 174 {
174 final Logger logger = LuanLogger.getLogger(luan,LuceneIndex.class); 175 this.luanLogger = luan.getLogger(LuceneIndex.class);
175 this.key = key; 176 this.key = key;
176 this.defaultFieldParser = defaultFieldParser; 177 this.defaultFieldParser = defaultFieldParser;
177 this.defaultFields = defaultFields; 178 this.defaultFields = defaultFields;
178 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); 179 mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
179 mfp.fields.put( "type", STRING_FIELD_PARSER ); 180 mfp.fields.put( "type", STRING_FIELD_PARSER );
189 if( postgresSpec == null ) { 190 if( postgresSpec == null ) {
190 postgresBackup = null; 191 postgresBackup = null;
191 } else { 192 } else {
192 Map spec = postgresSpec.asMap(); 193 Map spec = postgresSpec.asMap();
193 LuanFunction completer = Utils.removeRequiredFunction(spec,"completer"); 194 LuanFunction completer = Utils.removeRequiredFunction(spec,"completer");
194 postgresBackup = new PostgresBackup(spec); 195 postgresBackup = new PostgresBackup(luan,spec);
195 if( postgresBackup != null ) { 196 if( postgresBackup != null ) {
196 if( !wasCreated && postgresBackup.wasCreated ) { 197 if( !wasCreated && postgresBackup.wasCreated ) {
197 logger.error("rebuilding postgres backup"); 198 luanLogger.error("rebuilding postgres backup");
198 rebuild_postgres_backup(completer); 199 rebuild_postgres_backup(completer);
199 } else if( wasCreated && !postgresBackup.wasCreated ) { 200 } else if( wasCreated && !postgresBackup.wasCreated ) {
200 logger.error("restoring from postgres"); 201 luanLogger.error("restoring from postgres");
201 restore_from_postgres(luan); 202 restore_from_postgres();
202 } 203 }
203 } 204 }
204 } 205 }
205 } 206 }
206 207
820 } 821 }
821 822
822 public void rebuild_postgres_backup(LuanFunction completer) 823 public void rebuild_postgres_backup(LuanFunction completer)
823 throws IOException, LuanException, SQLException 824 throws IOException, LuanException, SQLException
824 { 825 {
825 final Logger logger = LuanLogger.getLogger(completer.luan(),LuceneIndex.class); 826 luanLogger.info("start rebuild_postgres_backup");
826 logger.info("start rebuild_postgres_backup");
827 writeLock.lock(); 827 writeLock.lock();
828 IndexSearcher searcher = openSearcher(); 828 IndexSearcher searcher = openSearcher();
829 boolean ok = false; 829 boolean ok = false;
830 try { 830 try {
831 postgresBackup.begin(); 831 postgresBackup.begin();
856 close(searcher); 856 close(searcher);
857 if( !ok ) 857 if( !ok )
858 postgresBackup.rollback(); 858 postgresBackup.rollback();
859 writeLock.unlock(); 859 writeLock.unlock();
860 } 860 }
861 logger.info("end rebuild_postgres_backup"); 861 luanLogger.info("end rebuild_postgres_backup");
862 } 862 }
863 863
864 public void restore_from_postgres(Luan luan) 864 public void restore_from_postgres()
865 throws IOException, LuanException, SQLException 865 throws IOException, LuanException, SQLException
866 { 866 {
867 final Logger logger = LuanLogger.getLogger(luan,LuceneIndex.class); 867 luanLogger.warn("start restore_from_postgres");
868 logger.warn("start restore_from_postgres");
869 if( postgresBackup==null ) 868 if( postgresBackup==null )
870 throw new NullPointerException(); 869 throw new NullPointerException();
871 if( writeLock.isHeldByCurrentThread() ) 870 if( writeLock.isHeldByCurrentThread() )
872 throw new RuntimeException(); 871 throw new RuntimeException();
873 writeLock.lock(); 872 writeLock.lock();
885 reopen(); 884 reopen();
886 } 885 }
887 wrote(); 886 wrote();
888 writeLock.unlock(); 887 writeLock.unlock();
889 } 888 }
890 logger.warn("end restore_from_postgres"); 889 luanLogger.warn("end restore_from_postgres");
891 } 890 }
892 891
893 void restore(LuanTable doc) 892 void restore(LuanTable doc)
894 throws LuanException, IOException 893 throws LuanException, IOException
895 { 894 {
896 writer.addDocument(toLucene(doc,null)); 895 writer.addDocument(toLucene(doc,null));
897 } 896 }
898 897
899 public void check(LuanFunction completer) throws IOException, SQLException, LuanException { 898 public void check(LuanFunction completer) throws IOException, SQLException, LuanException {
900 final Logger logger = LuanLogger.getLogger(completer.luan(),LuceneIndex.class); 899 luanLogger.info("start check");
901 logger.info("start check");
902 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); 900 CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
903 if( !status.clean ) 901 if( !status.clean )
904 logger.error("index not clean"); 902 luanLogger.error("index not clean");
905 if( postgresBackup != null ) 903 if( postgresBackup != null )
906 checkPostgres(completer); 904 checkPostgres(completer);
907 logger.info("end check"); 905 luanLogger.info("end check");
908 } 906 }
909 907
910 private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException { 908 private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException {
911 final Logger logger = LuanLogger.getLogger(completer.luan(),LuceneIndex.class);
912 final PostgresBackup.Checker postgresChecker; 909 final PostgresBackup.Checker postgresChecker;
913 final IndexSearcher searcher; 910 final IndexSearcher searcher;
914 writeLock.lock(); 911 writeLock.lock();
915 try { 912 try {
916 postgresChecker = postgresBackup.newChecker(); 913 postgresChecker = postgresBackup.newChecker();
941 while( iLucene < nLucene && iPostgres < nPostgres ) { 938 while( iLucene < nLucene && iPostgres < nPostgres ) {
942 long idLucene = idsLucene.get(iLucene); 939 long idLucene = idsLucene.get(iLucene);
943 long idPostgres = idsPostgres.get(iPostgres); 940 long idPostgres = idsPostgres.get(iPostgres);
944 if( idLucene < idPostgres ) { 941 if( idLucene < idPostgres ) {
945 iLucene++; 942 iLucene++;
946 logger.error("id "+idLucene+" found in lucene but not postgres"); 943 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
947 } else if( idLucene > idPostgres ) { 944 } else if( idLucene > idPostgres ) {
948 iPostgres++; 945 iPostgres++;
949 logger.error("id "+idPostgres+" found in postgres but not lucene"); 946 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
950 } else { // == 947 } else { // ==
951 LuanTable docPostgres = postgresChecker.getDoc(idPostgres); 948 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
952 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); 949 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
953 if( td.totalHits != 1 ) throw new RuntimeException(); 950 if( td.totalHits != 1 ) throw new RuntimeException();
954 Document doc = searcher.doc( td.scoreDocs[0].doc ); 951 Document doc = searcher.doc( td.scoreDocs[0].doc );
955 LuanTable docLucene = toTable(completer.luan(),doc); 952 LuanTable docLucene = toTable(completer.luan(),doc);
956 docLucene = (LuanTable)completer.call(docLucene); 953 docLucene = (LuanTable)completer.call(docLucene);
957 if( !equal(docPostgres,docLucene) ) { 954 if( !equal(docPostgres,docLucene) ) {
958 logger.error("id "+idLucene+" not equal"); 955 luanLogger.error("id "+idLucene+" not equal");
959 logger.error("lucene = "+lts.toString(docLucene)); 956 luanLogger.error("lucene = "+lts.toString(docLucene));
960 logger.error("postgres = "+lts.toString(docPostgres)); 957 luanLogger.error("postgres = "+lts.toString(docPostgres));
961 } 958 }
962 iLucene++; 959 iLucene++;
963 iPostgres++; 960 iPostgres++;
964 } 961 }
965 } 962 }
966 while( iLucene < nLucene ) { 963 while( iLucene < nLucene ) {
967 long idLucene = idsLucene.get(iLucene++); 964 long idLucene = idsLucene.get(iLucene++);
968 logger.error("id "+idLucene+" found in lucene but not postgres"); 965 luanLogger.error("id "+idLucene+" found in lucene but not postgres");
969 } 966 }
970 while( iPostgres < nPostgres ) { 967 while( iPostgres < nPostgres ) {
971 long idPostgres = idsPostgres.get(iPostgres++); 968 long idPostgres = idsPostgres.get(iPostgres++);
972 logger.error("id "+idPostgres+" found in postgres but not lucene"); 969 luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
973 } 970 }
974 } finally { 971 } finally {
975 close(searcher); 972 close(searcher);
976 postgresChecker.close(); 973 postgresChecker.close();
977 } 974 }