Mercurial Hosting > luan

changeset 1696:2958cf04d844

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
remove postgres backup
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 26 Jun 2022 14:40:08 -0600
parents 25833dd89844
children aff2309ae510
files examples/blog/src/lib/Db.luan src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java
diffstat 4 files changed, 10 insertions(+), 515 deletions(-) [+]
line wrap: on
line diff
--- a/examples/blog/src/lib/Db.luan	Mon Jun 20 16:37:23 2022 -0600
+++ b/examples/blog/src/lib/Db.luan	Sun Jun 26 14:40:08 2022 -0600
@@ -17,21 +17,16 @@
 	BackupIndexWriter.backupDomains = {"localhost"}
 end
 
---local postgres_spec = Hosted.postgres_spec()
---logger.info(stringify(postgres_spec))
-
 local dir = uri("site:/private/local/lucene")
 local Db = Lucene.index( dir, {
 	default_type = Lucene.type.english
 	default_fields = {"subject","content"}
-	--postgres_spec = postgres_spec
 	log_dir = uri("site:/private/local/lucene_log")
 } )
 
 --	this is how you index a field
 --	db.indexed_fields.post_date = Lucene.type.long
 
---Db.restore_from_postgres()
 Db.restore_from_log()
 Db.update{
 	[1] = function()
--- a/src/luan/modules/lucene/Lucene.luan	Mon Jun 20 16:37:23 2022 -0600
+++ b/src/luan/modules/lucene/Lucene.luan	Sun Jun 26 14:40:08 2022 -0600
@@ -113,10 +113,6 @@
 	index.restore_from_log = java_index.restore_from_log
 	index.force_restore_from_log = java_index.force_restore_from_log
 
-	index.has_postgres_backup = java_index.hasPostgresBackup()
-	index.rebuild_postgres_backup = java_index.rebuild_postgres_backup
-	index.restore_from_postgres = java_index.restore_from_postgres
-	index.force_restore_from_postgres = java_index.force_restore_from_postgres
 	index.check = java_index.check
 
 	function index.not_in_transaction() end
--- a/src/luan/modules/lucene/LuceneIndex.java	Mon Jun 20 16:37:23 2022 -0600
+++ b/src/luan/modules/lucene/LuceneIndex.java	Sun Jun 26 14:40:08 2022 -0600
@@ -7,7 +7,6 @@
 import java.io.IOException;
 import java.lang.ref.Reference;
 import java.lang.ref.WeakReference;
-import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.Map;
@@ -100,7 +99,7 @@
 	private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>();
 
 	public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options)
-		throws LuanException, IOException, ClassNotFoundException, SQLException
+		throws LuanException, IOException, ClassNotFoundException
 	{
 		String key = indexDir.getCanonicalPath();
 		synchronized(indexes) {
@@ -143,7 +142,6 @@
 	private AtomicInteger writeCounter = new AtomicInteger();
 	private final GoodIndexWriterConfig config;
 
-	private final PostgresBackup postgresBackup;
 	private boolean wasCreated;
 	private final File logDir;
 	private final long logTime;
@@ -151,14 +149,13 @@
 	private final String domain;
 
 	private LuceneIndex(Luan luan,File indexDir,LuanTable options)
-		throws LuanException, IOException, ClassNotFoundException, SQLException
+		throws LuanException, IOException, ClassNotFoundException
 	{
 		options = new LuanTable(options);
 		this.version = options.remove("version");
 		FieldParser defaultFieldParser = (FieldParser)options.remove("default_type");
 		LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
 		String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
-		LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
 		LuanFunction supplementer = Utils.removeFunction(options,"supplementer");
 		logDir = (File)options.remove("log_dir");
 		logTime = (Long)options.remove("log_time");
@@ -185,20 +182,6 @@
 		this.analyzer = analyzer;
 		this.config = new SupplementingConfig(luceneVersion,mfp,supplementer);
 		wasCreated = reopen();
-		if( postgresSpec == null ) {
-			postgresBackup = null;
-		} else {
-			postgresBackup = new PostgresBackup(postgresSpec);
-			if( !wasCreated && postgresBackup.wasCreated ) {
-				logger.error("rebuilding postgres backup");
-				rebuild_postgres_backup(luan);
-/*
-			} else if( wasCreated && !postgresBackup.wasCreated ) {
-				logger.error("restoring from postgres");
-				restore_from_postgres();
-*/
-			}
-		}
 	}
 
 	public boolean reopen() throws IOException {
@@ -222,14 +205,12 @@
 		writeCounter.incrementAndGet();
 	}
 
-	public void delete_all() throws IOException, SQLException {
+	public void delete_all() throws IOException {
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
 		try {
 			writer.deleteAll();
 			id = 0;
-			if( postgresBackup != null )
-				postgresBackup.deleteAll();
 			if(commit) writer.commit();
 		} finally {
 			wrote();
@@ -243,43 +224,14 @@
 		return new Term(key,br);
 	}
 
-	private static final Set<String> ID_SET = Collections.singleton("id");
-
-	private void backupDelete(Query query)
-		throws IOException, SQLException, LuanException
-	{
-		if( postgresBackup != null ) {
-			final List<Long> ids = new ArrayList<Long>();
-			IndexSearcher searcher = openSearcher();
-			try {
-				MyCollector col = new MyCollector() {
-					@Override public void collect(int iDoc) throws IOException {
-						Document doc = searcher.doc( docBase + iDoc, ID_SET );
-						Long id = (Long)doc.getField("id").numericValue();
-						ids.add(id);
-					}
-				};
-				searcher.search(query,col);
-			} finally {
-				close(searcher);
-			}
-			postgresBackup.begin();
-			for( Long id : ids ) {
-				postgresBackup.delete(id);
-			}
-			postgresBackup.commit();
-		}
-	}
-
 	public void delete(String queryStr)
-		throws IOException, ParseException, SQLException, LuanException
+		throws IOException, ParseException, LuanException
 	{
 		Query query = GoodQueryParser.parseQuery(mfp,queryStr);
 
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
 		try {
-			backupDelete(query);
 			writer.deleteDocuments(query);
 			if(commit) writer.commit();
 		} finally {
@@ -305,7 +257,7 @@
 	}
 
 	public void save( Luan luan, LuanTable doc, LuanTable unstored, Map<String,Float> boosts )
-		throws LuanException, IOException, SQLException
+		throws LuanException, IOException
 	{
 		Object obj = doc.get(luan,"id");
 		Long id;
@@ -325,8 +277,6 @@
 					throw new LuanException("boosts required with unstored");
 				if( id != null )
 					throw new LuanException("update not supported");
-				if( postgresBackup != null )
-					throw new LuanException("not supported with postgres backup");
 				if( !(writer instanceof LuceneIndexWriter) )
 					throw new LuanException("not supported with index logging");
 				id = ++this.id;
@@ -336,12 +286,8 @@
 			} else if( id == null ) {
 				id = ++this.id;
 				doc.put(luan,"id",id);
-				if( postgresBackup != null )
-					postgresBackup.add(luan,doc);
 				writer.addDocument(toLucene(doc));
 			} else {
-				if( postgresBackup != null )
-					postgresBackup.update(luan,doc);
 				writer.updateDocument( "id", toLucene(doc) );
 			}
 			if(commit) writer.commit();
@@ -356,26 +302,20 @@
 	}
 
 	public Object run_in_transaction(Luan luan,LuanFunction fn)
-		throws IOException, LuanException, SQLException
+		throws IOException, LuanException
 	{
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
 		boolean ok = false;
 		try {
-			if( commit && postgresBackup != null )
-				postgresBackup.begin();
 			Object rtn = fn.call(luan);
 			ok = true;
 			if(commit) {
-				if( postgresBackup != null )
-					postgresBackup.commit();
 				writer.commit();
 			}
 			return rtn;
 		} finally {
 			if( !ok && commit ) {
-				if( postgresBackup != null )
-					postgresBackup.rollback();
 				writer.rollback();
 				reopen();
 			}
@@ -474,17 +414,15 @@
 		super.finalize();
 	}
 
-	public void close() throws IOException, SQLException {
+	public void close() throws IOException {
 		closeWriter();
 		reader.close();
 	}
 
-	private void closeWriter() throws IOException, SQLException {
+	private void closeWriter() throws IOException {
 		writeLock.lock();
 		try {
 			writer.close();
-			if( postgresBackup != null )
-				postgresBackup.close();
 		} finally {
 			writeLock.unlock();
 		}
@@ -717,91 +655,6 @@
 	}
 
 
-
-	public boolean hasPostgresBackup() {
-		return postgresBackup != null;
-	}
-
-	public void rebuild_postgres_backup(Luan luan)
-		throws IOException, LuanException, SQLException
-	{
-		logger.info("start rebuild_postgres_backup");
-		writeLock.lock();
-		IndexSearcher searcher = openSearcher();
-		boolean ok = false;
-		try {
-			postgresBackup.begin();
-			postgresBackup.deleteAll();
-			Query query = new PrefixQuery(new Term("id"));
-			MyCollector col = new MyCollector() {
-				@Override public void collect(int iDoc) throws IOException {
-					try {
-						Document doc = searcher.doc( docBase + iDoc );
-						LuanTable tbl = toTable(doc);
-						postgresBackup.add(luan,tbl);
-					} catch(LuanException e) {
-						throw new LuanRuntimeException(e);
-					} catch(SQLException e) {
-						throw new RuntimeException(e);
-					}
-				}
-			};
-			try {
-				searcher.search(query,col);
-			} catch(LuanRuntimeException e) {
-				throw (LuanException)e.getCause();
-			}
-			ok = true;
-			postgresBackup.commit();
-		} finally {
-			close(searcher);
-			if( !ok )
-				postgresBackup.rollback();
-			writeLock.unlock();
-		}
-		logger.info("end rebuild_postgres_backup");
-	}
-
-	public void restore_from_postgres()
-		throws IOException, LuanException, SQLException, ParseException
-	{
-		if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
-			logger.error("restoring from postgres");
-			force_restore_from_postgres();
-		}
-	}
-
-	public void force_restore_from_postgres()
-		throws IOException, LuanException, SQLException, ParseException
-	{
-		logger.warn("start restore_from_postgres");
-		if( postgresBackup==null )
-			throw new NullPointerException();
-		if( writeLock.isHeldByCurrentThread() )
-			throw new RuntimeException();
-		writeLock.lock();
-		boolean ok = false;
-		try {
-			writer.tag("restore_from_postgres");
-			writer.deleteAll();
-			postgresBackup.restoreLucene(this);
-			ok = true;
-			writer.commit();
-			wrote();
-			ensure_open();  // refresh searcher
-			initId();
-			wasCreated = false;
-		} finally {
-			if( !ok ) {
-				writer.rollback();
-				reopen();
-			}
-			wrote();
-			writeLock.unlock();
-		}
-		logger.warn("end restore_from_postgres");
-	}
-
 	void restore(LuanTable doc)
 		throws LuanException, IOException
 	{
@@ -823,7 +676,7 @@
 	}
 
 	public void restore_from_log(Luan luan,LuanFunction handler)
-		throws IOException, LuanException, SQLException, ParseException
+		throws IOException, LuanException, ParseException
 	{
 		LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer;
 		if( wasCreated && !loggingWriter.wasCreated ) {
@@ -860,11 +713,8 @@
 		logger.warn("end force_restore_from_log");
 	}
 
-	public void check() throws IOException, SQLException, LuanException, ParseException {
-		boolean hasPostgres = postgresBackup != null;
+	public void check() throws IOException, LuanException, ParseException {
 		String msg = "start check";
-		if( hasPostgres )
-			msg += " with postgres";
 		logger.info(msg);
 		CheckIndex.Status status = new CheckIndex(fsDir).checkIndex();
 		if( !status.clean )
@@ -874,137 +724,7 @@
 			logger.info("log check");
 			boolean ok = loggingWriter.check(ID_SORT);
 		}
-		if( hasPostgres ) {
-			logger.info("postgres check");
-			checkPostgres();
-		}
 		logger.info("end check");
 	}
 
-	private void checkPostgres()
-		throws IOException, SQLException, LuanException, ParseException
-	{
-		final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker();
-		final IndexSearcher searcher = openSearcher();
-		try {
-			final List<Long> idsLucene = new ArrayList<Long>();
-			Query query = new PrefixQuery(new Term("id"));
-			MyCollector col = new MyCollector() {
-				@Override public void collect(int iDoc) throws IOException {
-					Document doc = searcher.doc( docBase + iDoc );
-					Long id = (Long)doc.getField("id").numericValue();
-					idsLucene.add(id);
-				}
-			};
-			searcher.search(query,col);
-			Collections.sort(idsLucene);
-			final List<Long> idsPostgres = postgresChecker.getIds();
-			final int nLucene = idsLucene.size();
-			final int nPostgres = idsPostgres.size();
-			int iLucene = 0;
-			int iPostgres = 0;
-			LuanToString lts = new LuanToString();
-			lts.settingsInit.strict = true;
-			lts.settingsInit.numberTypes = true;
-			while( iLucene < nLucene && iPostgres < nPostgres ) {
-				long idLucene = idsLucene.get(iLucene);
-				long idPostgres = idsPostgres.get(iPostgres);
-				if( idLucene < idPostgres ) {
-					iLucene++;
-					checkPostgres(postgresChecker,lts,idLucene);
-				} else if( idLucene > idPostgres ) {
-					iPostgres++;
-					checkPostgres(postgresChecker,lts,idPostgres);
-				} else {  // ==
-					LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
-					TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
-					if( td.totalHits != 1 )  throw new RuntimeException();
-					Document doc = searcher.doc( td.scoreDocs[0].doc );
-					LuanTable docLucene = toTable(doc);
-					if( !equal(docPostgres,docLucene) ) {
-						checkPostgres(postgresChecker,lts,idPostgres);
-					}
-					iLucene++;
-					iPostgres++;
-				}
-			}
-			while( iLucene < nLucene ) {
-				long idLucene = idsLucene.get(iLucene++);
-				checkPostgres(postgresChecker,lts,idLucene);
-			}
-			while( iPostgres < nPostgres ) {
-				long idPostgres = idsPostgres.get(iPostgres++);
-				checkPostgres(postgresChecker,lts,idPostgres);
-			}
-		} finally {
-			close(searcher);
-			postgresChecker.close();
-		}
-	}
-
-	private void checkPostgres(PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
-		throws IOException, SQLException, LuanException, ParseException
-	{
-		//logger.info("check id "+id);
-		writeLock.lock();
-		try {
-			final IndexSearcher searcher = openSearcher();
-			try {
-				LuanTable docPostgres = postgresChecker.getDoc(id);
-				TopDocs td = searcher.search(new TermQuery(term("id",id)),1);
-				LuanTable docLucene;
-				if( td.totalHits == 0 )  {
-					docLucene = null;
-				} else if( td.totalHits == 1 ) {
-					Document doc = searcher.doc( td.scoreDocs[0].doc );
-					docLucene = toTable(doc);
-				} else
-					throw new RuntimeException();
-				if( docPostgres == null ) {
-					if( docLucene != null )
-						logger.error("id "+id+" found in lucene but not postgres");
-					return;
-				}
-				if( docLucene == null ) {
-					logger.error("id "+id+" found in postgres but not lucene");
-					return;
-				}
-				if( !equal(docPostgres,docLucene) ) {
-					logger.error("id "+id+" not equal");
-					logger.error("lucene = "+lts.toString(docLucene));
-					logger.error("postgres = "+lts.toString(docPostgres));
-				}
-			} finally {
-				close(searcher);
-			}
-		} finally {
-			writeLock.unlock();
-		}
-	}
-
-	private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
-		return t1!=null && t2!=null && toJava(t1).equals(toJava(t2));
-	}
-
-	private static Map toJava(LuanTable t) throws LuanException {
-		Map map = t.asMap();
-		for( Iterator iter = map.entrySet().iterator(); iter.hasNext(); ) {
-			Map.Entry entry = (Map.Entry)iter.next();
-			Object value = entry.getValue();
-			if( value instanceof LuanTable ) {
-				LuanTable v = (LuanTable)value;
-				if( !v.isList() )
-					logger.error("not list");
-				List list = v.asList();
-				if( list.isEmpty() ) {
-					iter.remove();
-				} else if( list.size() == 1 ) {
-					entry.setValue(list.get(0));
-				} else {
-					entry.setValue(list);
-				}
-			}
-		}
-		return map;
-	}
 }
--- a/src/luan/modules/lucene/PostgresBackup.java	Mon Jun 20 16:37:23 2022 -0600
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,216 +0,0 @@
-package luan.modules.lucene;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.Statement;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.ResultSet;
-import java.util.Properties;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import luan.Luan;
-import luan.LuanTable;
-import luan.LuanFunction;
-import luan.LuanException;
-import luan.modules.Utils;
-import luan.modules.parsers.LuanToString;
-import luan.modules.parsers.LuanParser;
-import goodjava.parser.ParseException;
-import goodjava.logging.Logger;
-import goodjava.logging.LoggerFactory;
-
-
-final class PostgresBackup {
-	private static final Logger logger = LoggerFactory.getLogger(PostgresBackup.class);
-
-	final boolean wasCreated;
-	private final String url;
-	private final Properties props = new Properties();
-	private final Connection con;
-	private final PreparedStatement insertStmt;
-	private final PreparedStatement updateStmt;
-	private final PreparedStatement deleteStmt;
-	private int trans = 0;
-	private final LuanToString luanToString;
-
-	PostgresBackup(LuanTable spec)
-		throws ClassNotFoundException, SQLException, LuanException
-	{
-		spec = new LuanTable(spec);
-/*
-		Class.forName("org.postgresql.Driver");
-		url = "jdbc:postgresql://localhost:5432/luan";
-		props.setProperty("user","postgres");
-		props.setProperty("password","");
-*/
-		String cls = "org.postgresql.Driver";
-		if( !Utils.removeRequiredString(spec,"class").equals(cls) )
-			throw new LuanException( "parameter 'class' must be '"+cls+"'" );
-		Class.forName(cls);
-		url = Utils.removeRequiredString(spec,"url");
-		props.setProperty( "user", Utils.removeRequiredString(spec,"user") );
-		props.setProperty( "password", Utils.removeRequiredString(spec,"password") );
-		Utils.checkEmpty(spec);
-
-		con = newConnection();
-
-		Statement stmt = con.createStatement();
-		boolean hasTable = stmt.executeQuery(
-			"select * from information_schema.tables where table_name='lucene'"
-		).next();
-		if( !hasTable ) {
-			stmt.executeUpdate(
-				"create table lucene ("
-				+"	id integer not null primary key,"
-				+"	data text not null"
-				+")"
-			);
-		}
-		stmt.close();
-		wasCreated = !hasTable;
-
-		insertStmt = con.prepareStatement(
-			"insert into lucene (id,data) values (?,?)"
-		);
-		updateStmt = con.prepareStatement(
-			"update lucene set data=? where id=?"
-		);
-		deleteStmt = con.prepareStatement(
-			"delete from lucene where id=?"
-		);
-
-		luanToString = new LuanToString();
-		luanToString.settingsInit.strict = true;
-		luanToString.settingsInit.numberTypes = true;
-	}
-
-	Connection newConnection() throws SQLException {
-		return DriverManager.getConnection(url,props);
-	}
-
-	void close() throws SQLException {
-		insertStmt.close();
-		updateStmt.close();
-		deleteStmt.close();
-		con.close();
-	}
-
-	protected void finalize() throws Throwable {
-		close();
-		super.finalize();
-	}
-
-	void add(Luan luan,LuanTable doc) throws LuanException, SQLException {
-		Long id = (Long)doc.get(luan,"id");
-		String data = luanToString.toString(doc);
-		insertStmt.setLong(1,id);
-		insertStmt.setString(2,data);
-		insertStmt.executeUpdate();
-	}
-
-	void update(Luan luan,LuanTable doc) throws LuanException, SQLException {
-		Long id = (Long)doc.get(luan,"id");
-		String data = luanToString.toString(doc);
-		updateStmt.setString(1,data);
-		updateStmt.setLong(2,id);
-		int n = updateStmt.executeUpdate();
-		if( n==0 ) {
-			logger.error("update not found for id="+id+", trying add");
-			add(luan,doc);
-		} else if( n!=1 )
-			throw new RuntimeException();
-	}
-
-	void deleteAll() throws SQLException {
-		Statement stmt = con.createStatement();
-		stmt.executeUpdate("delete from lucene");
-		stmt.close();
-	}
-
-	void delete(long id) throws SQLException, LuanException {
-		deleteStmt.setLong(1,id);
-		int n = deleteStmt.executeUpdate();
-		if( n==0 )
-			throw new LuanException("delete not found for id="+id);
-	}
-
-	void begin() throws SQLException {
-		if( trans++ == 0 )
-			con.setAutoCommit(false);
-	}
-
-	void commit() throws SQLException, LuanException {
-		if( trans <= 0 )
-			throw new LuanException("commit not in transaction");
-		if( --trans == 0 )
-			con.setAutoCommit(true);
-	}
-
-	void rollback() throws SQLException, LuanException {
-		if( --trans != 0 )
-			throw new LuanException("rollback failed trans="+trans);
-		con.rollback();
-		con.setAutoCommit(true);
-	}
-
-	void restoreLucene(LuceneIndex li)
-		throws LuanException, IOException, SQLException, ParseException
-	{
-		Statement stmt = con.createStatement();
-		ResultSet rs = stmt.executeQuery("select data from lucene");
-		while( rs.next() ) {
-			String data = rs.getString("data");
-			LuanTable doc = (LuanTable)LuanParser.parse(data);
-			li.restore(doc);
-		}
-		stmt.close();
-	}
-
-	final class Checker {
-		private final Connection con;
-		private final PreparedStatement pstmt;
-
-		Checker() throws SQLException {
-			con = newConnection();
-			con.setReadOnly(true);
-			pstmt = con.prepareStatement(
-				"select data from lucene where id=?"
-			);
-		}
-
-		void close() throws SQLException {
-			pstmt.close();
-			con.close();
-		}
-
-		List<Long> getIds() throws SQLException {
-			List<Long> ids = new ArrayList<Long>();
-			Statement stmt = con.createStatement();
-			ResultSet rs = stmt.executeQuery("select id from lucene order by id");
-			while( rs.next() ) {
-				long id = rs.getLong("id");
-				ids.add(id);
-			}
-			stmt.close();
-			return ids;
-		}
-
-		LuanTable getDoc(long id) throws SQLException, ParseException {
-			pstmt.setLong(1,id);
-			ResultSet rs = pstmt.executeQuery();
-			if( !rs.next() )
-				return null;
-			String data = rs.getString("data");
-			LuanTable doc = (LuanTable)LuanParser.parse(data);
-			return doc;
-		}
-	}
-
-	Checker newChecker() throws SQLException {
-		return new Checker();
-	}
-
-}