Mercurial Hosting > luan

changeset 1430:103d0ce70385

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix postgres check
author Franklin Schmidt <fschmidt@gmail.com>
date Sat, 30 Nov 2019 21:32:14 -0700
parents 82415c9c0015
children f3a417074cea
files src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java
diffstat 2 files changed, 51 insertions(+), 26 deletions(-) [+]
line wrap: on
line diff
--- a/src/luan/modules/lucene/LuceneIndex.java	Sun Nov 24 23:07:21 2019 -0700
+++ b/src/luan/modules/lucene/LuceneIndex.java	Sat Nov 30 21:32:14 2019 -0700
@@ -904,17 +904,12 @@
 		luanLogger.info("end check");
 	}
 
-	private void checkPostgres(LuanFunction completer) throws IOException, SQLException, LuanException {
+	private void checkPostgres(LuanFunction completer)
+		throws IOException, SQLException, LuanException
+	{
 		//luanLogger.info("start postgres check");
-		final PostgresBackup.Checker postgresChecker;
-		final IndexSearcher searcher;
-		writeLock.lock();
-		try {
-			postgresChecker = postgresBackup.newChecker();
-			searcher = openSearcher();
-		} finally {
-			writeLock.unlock();
-		}
+		final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
+		final IndexSearcher searcher = openSearcher();
 		try {
 			final List<Long> idsLucene = new ArrayList<Long>();
 			Query query = new PrefixQuery(new Term("id"));
@@ -940,10 +935,10 @@
 				long idPostgres = idsPostgres.get(iPostgres);
 				if( idLucene < idPostgres ) {
 					iLucene++;
-					luanLogger.error("id "+idLucene+" found in lucene but not postgres");
+					checkPostgres(completer,postgresChecker,lts,idLucene);
 				} else if( idLucene > idPostgres ) {
 					iPostgres++;
-					luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
+					checkPostgres(completer,postgresChecker,lts,idPostgres);
 				} else {  // ==
 					LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
 					TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
@@ -952,9 +947,7 @@
 					LuanTable docLucene = toTable(completer.luan(),doc);
 					docLucene = (LuanTable)completer.call(docLucene);
 					if( !equal(docPostgres,docLucene) ) {
-						luanLogger.error("id "+idLucene+" not equal");
-						luanLogger.error("lucene = "+lts.toString(docLucene));
-						luanLogger.error("postgres = "+lts.toString(docPostgres));
+						checkPostgres(completer,postgresChecker,lts,idPostgres);
 					}
 					iLucene++;
 					iPostgres++;
@@ -962,11 +955,11 @@
 			}
 			while( iLucene < nLucene ) {
 				long idLucene = idsLucene.get(iLucene++);
-				luanLogger.error("id "+idLucene+" found in lucene but not postgres");
+				checkPostgres(completer,postgresChecker,lts,idLucene);
 			}
 			while( iPostgres < nPostgres ) {
 				long idPostgres = idsPostgres.get(iPostgres++);
-				luanLogger.error("id "+idPostgres+" found in postgres but not lucene");
+				checkPostgres(completer,postgresChecker,lts,idPostgres);
 			}
 		} finally {
 			close(searcher);
@@ -974,6 +967,47 @@
 		}
 	}
 
+	private void checkPostgres(LuanFunction completer,PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
+		throws IOException, SQLException, LuanException
+	{
+		//luanLogger.info("check id "+id);
+		writeLock.lock();
+		try {
+			final IndexSearcher searcher = openSearcher();
+			try {
+				LuanTable docPostgres = postgresChecker.getDoc(id);
+				TopDocs td = searcher.search(new TermQuery(term("id",id)),1);
+				LuanTable docLucene;
+				if( td.totalHits == 0 )  {
+					docLucene = null;
+				} else if( td.totalHits == 1 ) {
+					Document doc = searcher.doc( td.scoreDocs[0].doc );
+					docLucene = toTable(completer.luan(),doc);
+					docLucene = (LuanTable)completer.call(docLucene);
+				} else
+					throw new RuntimeException();
+				if( docPostgres == null ) {
+					if( docLucene != null )
+						luanLogger.error("id "+id+" found in lucene but not postgres");
+					return;
+				}
+				if( docLucene == null ) {
+					luanLogger.error("id "+id+" found in postgres but not lucene");
+					return;
+				}
+				if( !equal(docPostgres,docLucene) ) {
+					luanLogger.error("id "+id+" not equal");
+					luanLogger.error("lucene = "+lts.toString(docLucene));
+					luanLogger.error("postgres = "+lts.toString(docPostgres));
+				}
+			} finally {
+				close(searcher);
+			}
+		} finally {
+			writeLock.unlock();
+		}
+	}
+
 	private boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
 		return t1!=null && t2!=null && t1.asMap().equals(t2.asMap());
 	}
--- a/src/luan/modules/lucene/PostgresBackup.java	Sun Nov 24 23:07:21 2019 -0700
+++ b/src/luan/modules/lucene/PostgresBackup.java	Sat Nov 30 21:32:14 2019 -0700
@@ -199,16 +199,7 @@
 
 		Checker() throws SQLException {
 			con = newConnection();
-			con.setAutoCommit(false);
 			con.setReadOnly(true);
-			con.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
-
-			// hack to capture data in transaction
-			Statement stmt = con.createStatement();
-			ResultSet rs = stmt.executeQuery("select 'x' from lucene");
-			while(rs.next());
-			stmt.close();
-
 			pstmt = con.prepareStatement(
 				"select data from lucene where id=?"
 			);