Mercurial Hosting > luan
changeset 1430:103d0ce70385
fix postgres check
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sat, 30 Nov 2019 21:32:14 -0700 |
parents | 82415c9c0015 |
children | f3a417074cea |
files | src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java |
diffstat | 2 files changed, 51 insertions(+), 26 deletions(-) [+] |
line wrap: on
line diff
diff -r 82415c9c0015 -r 103d0ce70385 src/luan/modules/lucene/LuceneIndex.java --- a/src/luan/modules/lucene/LuceneIndex.java Sun Nov 24 23:07:21 2019 -0700 +++ b/src/luan/modules/lucene/LuceneIndex.java Sat Nov 30 21:32:14 2019 -0700 @@ -904,17 +904,12 @@ luanLogger.info("end check"); } - private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException { + private void checkPostgres(LuanFunction completer) + throws IOException, SQLException, LuanException + { //luanLogger.info("start postgres check"); - final PostgresBackup.Checker postgresChecker; - final IndexSearcher searcher; - writeLock.lock(); - try { - postgresChecker = postgresBackup.newChecker(); - searcher = openSearcher(); - } finally { - writeLock.unlock(); - } + final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); + final IndexSearcher searcher = openSearcher(); try { final List<Long> idsLucene = new ArrayList<Long>(); Query query = new PrefixQuery(new Term("id")); @@ -940,10 +935,10 @@ long idPostgres = idsPostgres.get(iPostgres); if( idLucene < idPostgres ) { iLucene++; - luanLogger.error("id "+idLucene+" found in lucene but not postgres"); + checkPostgres(completer,postgresChecker,lts,idLucene); } else if( idLucene > idPostgres ) { iPostgres++; - luanLogger.error("id "+idPostgres+" found in postgres but not lucene"); + checkPostgres(completer,postgresChecker,lts,idPostgres); } else { // == LuanTable docPostgres = postgresChecker.getDoc(idPostgres); TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); @@ -952,9 +947,7 @@ LuanTable docLucene = toTable(completer.luan(),doc); docLucene = (LuanTable)completer.call(docLucene); if( !equal(docPostgres,docLucene) ) { - luanLogger.error("id "+idLucene+" not equal"); - luanLogger.error("lucene = "+lts.toString(docLucene)); - luanLogger.error("postgres = "+lts.toString(docPostgres)); + checkPostgres(completer,postgresChecker,lts,idPostgres); } iLucene++; iPostgres++; @@ -962,11 +955,11 @@ } while( iLucene < nLucene ) { long idLucene = idsLucene.get(iLucene++); - luanLogger.error("id "+idLucene+" found in lucene but not postgres"); + checkPostgres(completer,postgresChecker,lts,idLucene); } while( iPostgres < nPostgres ) { long idPostgres = idsPostgres.get(iPostgres++); - luanLogger.error("id "+idPostgres+" found in postgres but not lucene"); + checkPostgres(completer,postgresChecker,lts,idPostgres); } } finally { close(searcher); @@ -974,6 +967,47 @@ } } + private void checkPostgres(LuanFunction completer,PostgresBackup.Checker postgresChecker,LuanToString lts,long id) + throws IOException, SQLException, LuanException + { + //luanLogger.info("check id "+id); + writeLock.lock(); + try { + final IndexSearcher searcher = openSearcher(); + try { + LuanTable docPostgres = postgresChecker.getDoc(id); + TopDocs td = searcher.search(new TermQuery(term("id",id)),1); + LuanTable docLucene; + if( td.totalHits == 0 ) { + docLucene = null; + } else if( td.totalHits == 1 ) { + Document doc = searcher.doc( td.scoreDocs[0].doc ); + docLucene = toTable(completer.luan(),doc); + docLucene = (LuanTable)completer.call(docLucene); + } else + throw new RuntimeException(); + if( docPostgres == null ) { + if( docLucene != null ) + luanLogger.error("id "+id+" found in lucene but not postgres"); + return; + } + if( docLucene == null ) { + luanLogger.error("id "+id+" found in postgres but not lucene"); + return; + } + if( !equal(docPostgres,docLucene) ) { + luanLogger.error("id "+id+" not equal"); + luanLogger.error("lucene = "+lts.toString(docLucene)); + luanLogger.error("postgres = "+lts.toString(docPostgres)); + } + } finally { + close(searcher); + } + } finally { + writeLock.unlock(); + } + } + private boolean equal(LuanTable t1,LuanTable t2) throws LuanException { return t1!=null && t2!=null && t1.asMap().equals(t2.asMap()); }
diff -r 82415c9c0015 -r 103d0ce70385 src/luan/modules/lucene/PostgresBackup.java --- a/src/luan/modules/lucene/PostgresBackup.java Sun Nov 24 23:07:21 2019 -0700 +++ b/src/luan/modules/lucene/PostgresBackup.java Sat Nov 30 21:32:14 2019 -0700 @@ -199,16 +199,7 @@ Checker() throws SQLException { con = newConnection(); - con.setAutoCommit(false); con.setReadOnly(true); - con.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE); - - // hack to capture data in transaction - Statement stmt = con.createStatement(); - ResultSet rs = stmt.executeQuery("select 'x' from lucene"); - while(rs.next()); - stmt.close(); - pstmt = con.prepareStatement( "select data from lucene where id=?" );