Mercurial Hosting > luan

changeset 1387:bc40bc9aab3a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
start postgres backup
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 02 Sep 2019 22:23:12 -0600
parents dc36dd8bf839
children 2024d23ddd64
files examples/blog/src/delete.luan examples/blog/src/index.html.luan examples/blog/src/lib/Db.luan examples/blog/src/lib/Post.luan lib/postgresql-9.1-901.jdbc4.jar src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java
diffstat 8 files changed, 315 insertions(+), 8 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/examples/blog/src/delete.luan	Mon Sep 02 22:23:12 2019 -0600
@@ -0,0 +1,13 @@
+local Luan = require "luan:Luan.luan"
+local error = Luan.error
+local String = require "luan:String.luan"
+local to_number = String.to_number or error()
+local Http = require "luan:http/Http.luan"
+local Post = require "site:/lib/Post.luan"
+
+
+return function()
+	local post_id = to_number(Http.request.parameters.post) or error()
+	Post.delete_by_id(post_id)
+	Http.response.send_redirect("/")
+end
--- a/examples/blog/src/index.html.luan	Thu Aug 22 12:32:20 2019 -0600
+++ b/examples/blog/src/index.html.luan	Mon Sep 02 22:23:12 2019 -0600
@@ -39,7 +39,11 @@
 			%>
 			<a name="p<%= post.id %>">
 			<h2><%= post.subject %></h2>
-			<p><%= Time.format(post.date) %> - <a href="edit.html?post=<%= post.id %>">Edit</a></p>
+			<p>
+				<%= Time.format(post.date) %>
+				- <a href="edit.html?post=<%= post.id %>">Edit</a>
+				- <a href="delete?post=<%= post.id %>">Delete</a>
+			</p>
 			<pre><%= bbcode_to_html(html_encode(post.content)) %></pre>
 			<hr>
 			<%
--- a/examples/blog/src/lib/Db.luan	Thu Aug 22 12:32:20 2019 -0600
+++ b/examples/blog/src/lib/Db.luan	Mon Sep 02 22:23:12 2019 -0600
@@ -4,9 +4,13 @@
 
 local Db = {}
 
+local function completer(doc)
+	return doc
+end
+
 function Db.new(lucene_dir)
 	local dir = Io.uri(lucene_dir)
-	local db = Lucene.index( dir, Lucene.type.english, {"subject","content"} )
+	local db = Lucene.index( dir, Lucene.type.english, {"subject","content"}, completer )
 	
 --	this is how you index a field
 --	db.indexed_fields.post_date = Lucene.type.long
--- a/examples/blog/src/lib/Post.luan	Thu Aug 22 12:32:20 2019 -0600
+++ b/examples/blog/src/lib/Post.luan	Mon Sep 02 22:23:12 2019 -0600
@@ -67,4 +67,8 @@
 	return posts
 end
 
+function Post.delete_by_id(id)
+	db.delete("id:"..id)
+end
+
 return Post
Binary file lib/postgresql-9.1-901.jdbc4.jar has changed
--- a/src/luan/modules/lucene/Lucene.luan	Thu Aug 22 12:32:20 2019 -0600
+++ b/src/luan/modules/lucene/Lucene.luan	Mon Sep 02 22:23:12 2019 -0600
@@ -35,12 +35,12 @@
 
 Lucene.literal = SaneQueryParser.literal
 
-function Lucene.index(index_dir,default_type,default_fields)
+function Lucene.index(index_dir,default_type,default_fields,postgres_backup)
 	type(index_dir)=="table" or error "index_dir must be table"
 	index_dir.to_uri_string and matches(index_dir.to_uri_string(),"^file:") or error "must be file"
 	local index = {}
 	index.dir = index_dir
-	local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,default_type,default_fields)
+	local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,default_type,default_fields,postgres_backup)
 	index.java = java_index
 	index.closer = closer or error()
 
@@ -69,6 +69,9 @@
 	index.count_tokens = java_index.count_tokens
 	index.close = closer.close
 
+	index.has_postgres_backup = java_index.hasPostgresBackup()
+	index.rebuild_postgres_backup = java_index.rebuild_postgres_backup
+
 	function index.search( query, from, to, options )
 		from or error "missing 'from' parameter"
 		to or error "missing 'to' parameter"
--- a/src/luan/modules/lucene/LuceneIndex.java	Thu Aug 22 12:32:20 2019 -0600
+++ b/src/luan/modules/lucene/LuceneIndex.java	Mon Sep 02 22:23:12 2019 -0600
@@ -45,6 +45,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.Sort;
@@ -110,14 +111,14 @@
 
 	private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>();
 
-	public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields)
+	public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,LuanFunction completer)
 		throws LuanException, IOException
 	{
 		String key = indexDir.getCanonicalPath();
 		synchronized(indexes) {
 			LuceneIndex li = indexes.get(key);
 			if( li == null ) {
-				li = new LuceneIndex(indexDir,defaultFieldParser,defaultFields,key);
+				li = new LuceneIndex(indexDir,defaultFieldParser,defaultFields,key,completer);
 				li.openCount = 1;
 				indexes.put(key,li);
 			} else {
@@ -157,7 +158,9 @@
 	private final FieldParser defaultFieldParser;
 	private final String[] defaultFields;
 
-	private LuceneIndex(File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key)
+	private final PostgresBackup postgresBackup;
+
+	private LuceneIndex(File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanFunction completer)
 		throws LuanException, IOException
 	{
 		this.key = key;
@@ -174,6 +177,9 @@
 		}
 		this.analyzer = analyzer;
 		reopen();
+		postgresBackup = completer!=null ? PostgresBackup.newInstance() : null;
+		if( postgresBackup != null && postgresBackup.wasCreated )
+			rebuild_postgres_backup(completer);
 	}
 
 	public void reopen() throws LuanException, IOException {
@@ -199,6 +205,8 @@
 		try {
 			writer.deleteAll();
 			id = idLim = 0;
+			if( postgresBackup != null )
+				postgresBackup.deleteAll();
 			if(commit) writer.commit();
 		} finally {
 			wrote();
@@ -212,6 +220,28 @@
 		return new Term(key,br);
 	}
 
+	private void backupDelete(Query query)
+		throws IOException
+	{
+		if( postgresBackup != null ) {
+			final List<Long> ids = new ArrayList<Long>();
+			IndexSearcher searcher = openSearcher();
+			MyCollector col = new MyCollector() {
+				@Override public void collect(int iDoc) throws IOException {
+					Document doc = searcher.doc( docBase + iDoc );
+					Long id = (Long)doc.getField("id").numericValue();
+					ids.add(id);
+				}
+			};
+			searcher.search(query,col);
+			postgresBackup.begin();
+			for( Long id : ids ) {
+				postgresBackup.delete(id);
+			}
+			postgresBackup.commit();
+		}
+	}
+
 	public void delete(String queryStr)
 		throws IOException, ParseException
 	{
@@ -220,6 +250,7 @@
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
 		try {
+			backupDelete(query);
 			writer.deleteDocuments(query);
 			if(commit) writer.commit();
 		} finally {
@@ -235,6 +266,9 @@
 	public void save(LuanTable doc,LuanTable boosts)
 		throws LuanException, IOException
 	{
+		if( boosts!=null && postgresBackup!=null )
+			logger.error("boosts are not saved to postgres backup");
+
 		Object obj = doc.get("id");
 		Long id;
 		try {
@@ -250,8 +284,12 @@
 				id = nextId();
 				doc.put("id",id);
 				writer.addDocument(toLucene(doc,boosts));
+				if( postgresBackup != null )
+					postgresBackup.add(id,doc);
 			} else {
 				writer.updateDocument( term("id",id), toLucene(doc,boosts) );
+				if( postgresBackup != null )
+					postgresBackup.update(id,doc);
 			}
 			if(commit) writer.commit();
 		} finally {
@@ -263,16 +301,31 @@
 	public Object run_in_transaction(LuanFunction fn) throws IOException, LuanException {
 		boolean commit = !writeLock.isHeldByCurrentThread();
 		writeLock.lock();
+		boolean ok = false;
 		try {
+			if( commit && postgresBackup != null )
+				postgresBackup.begin();
 			Object rtn = fn.call();
-			if(commit) writer.commit();
+			ok = true;
+			if(commit) {
+				if( postgresBackup != null )
+					postgresBackup.commit();
+				writer.commit();
+			}
 			return rtn;
 		} finally {
+			if( !ok && commit ) {
+				if( postgresBackup != null )
+					postgresBackup.rollback();
+				writer.rollback();
+				reopen();
+			}
 			wrote();
 			writeLock.unlock();
 		}
 	}
 
+	// ???
 	public Object run_in_lock(LuanFunction fn) throws IOException, LuanException {
 		if( writeLock.isHeldByCurrentThread() )
 			throw new RuntimeException();
@@ -373,6 +426,8 @@
 	}
 
 	public void doClose() throws IOException {
+		if( postgresBackup != null )
+			postgresBackup.close();
 		writer.close();
 		reader.close();
 	}
@@ -725,4 +780,48 @@
 		return n;
 	}
 
+
+
+	public boolean hasPostgresBackup() {
+		return postgresBackup != null;
+	}
+
+	public void rebuild_postgres_backup(LuanFunction completer)
+		throws IOException, LuanException
+	{
+		writeLock.lock();
+		boolean ok = false;
+		try {
+			postgresBackup.begin();
+			postgresBackup.deleteAll();
+			Query query = new PrefixQuery(new Term("id"));
+			IndexSearcher searcher = openSearcher();
+			MyCollector col = new MyCollector() {
+				@Override public void collect(int iDoc) throws IOException {
+					try {
+						Document doc = searcher.doc( docBase + iDoc );
+						LuanTable tbl = toTable(completer.luan(),doc);
+						tbl = (LuanTable)completer.call(tbl);
+						Long id = (Long)tbl.get("id");
+						//logger.info("id = "+id);
+						postgresBackup.add(id,tbl);
+					} catch(LuanException e) {
+						throw new LuanRuntimeException(e);
+					}
+				}
+			};
+			try {
+				searcher.search(query,col);
+			} catch(LuanRuntimeException e) {
+				throw (LuanException)e.getCause();
+			}
+			ok = true;
+			postgresBackup.commit();
+		} finally {
+			if( !ok )
+				postgresBackup.rollback();
+			writeLock.unlock();
+		}
+	}
+
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/luan/modules/lucene/PostgresBackup.java	Mon Sep 02 22:23:12 2019 -0600
@@ -0,0 +1,180 @@
+package luan.modules.lucene;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Properties;
+import luan.LuanTable;
+import luan.LuanException;
+import luan.modules.parsers.LuanToString;
+import luan.lib.logging.Logger;
+import luan.lib.logging.LoggerFactory;
+
+
+final class PostgresBackup {
+	private static final Logger logger = LoggerFactory.getLogger(PostgresBackup.class);
+
+	static PostgresBackup newInstance() {
+		try {
+			return new PostgresBackup();
+		} catch(ClassNotFoundException e) {
+			logger.error("creation failed",e);
+			return null;
+		} catch(SQLException e) {
+			logger.error("creation failed",e);
+			return null;
+		}
+	}
+
+	final boolean wasCreated;
+	private final Connection con;
+	private final PreparedStatement insertStmt;
+	private final PreparedStatement updateStmt;
+	private final PreparedStatement deleteStmt;
+	private int trans = 0;
+
+	private PostgresBackup()
+		throws ClassNotFoundException, SQLException
+	{
+		Class.forName("org.postgresql.Driver");
+
+		String url = "jdbc:postgresql://localhost:5432/luan";
+		Properties props = new Properties();
+		props.setProperty("user","postgres");
+		props.setProperty("password","");
+
+		con = DriverManager.getConnection(url,props);
+
+		Statement stmt = con.createStatement();
+		boolean hasTable = stmt.executeQuery(
+			"select * from information_schema.tables where table_name='lucene'"
+		).next();
+		if( !hasTable ) {
+			stmt.executeUpdate(
+				"create table lucene ("
+				+"	id integer not null primary key,"
+				+"	data text not null"
+				+")"
+			);
+		}
+		stmt.close();
+		wasCreated = !hasTable;
+
+		insertStmt = con.prepareStatement(
+			"insert into lucene (id,data) values (?,?)"
+		);
+		updateStmt = con.prepareStatement(
+			"update lucene set data=? where id=?"
+		);
+		deleteStmt = con.prepareStatement(
+			"delete from lucene where id=?"
+		);
+	}
+
+	void close() {
+		try {
+			insertStmt.close();
+			updateStmt.close();
+			deleteStmt.close();
+			con.close();
+		} catch(SQLException e) {
+			logger.error("close failed",e);
+		}
+	}
+
+	protected void finalize() throws Throwable {
+		super.finalize();
+		if( !con.isClosed() ) {
+			logger.error("con not closed");
+			con.close();
+		}
+	}
+
+	void add(long id,LuanTable doc) throws LuanException {
+		try {
+//logger.info("getAutoCommit="+con.getAutoCommit());
+			String data = LuanToString.toString(doc,true);
+			insertStmt.setLong(1,id);
+			insertStmt.setString(2,data);
+			insertStmt.executeUpdate();
+		} catch(SQLException e) {
+			logger.error("add failed",e);
+		}
+	}
+
+	void update(long id,LuanTable doc) throws LuanException {
+		try {
+			String data = LuanToString.toString(doc,true);
+			updateStmt.setString(1,data);
+			updateStmt.setLong(2,id);
+			int n = updateStmt.executeUpdate();
+			if( n==0 ) {
+				logger.error("update not found for id="+id+", trying add");
+				add(id,doc);
+			} else if( n!=1 )
+				throw new RuntimeException();
+		} catch(SQLException e) {
+			logger.error("update failed",e);
+		}
+	}
+
+	void deleteAll() {
+		try {
+			Statement stmt = con.createStatement();
+			stmt.executeUpdate("delete from lucene");
+			stmt.close();
+		} catch(SQLException e) {
+			logger.error("update failed",e);
+		}
+	}
+
+	void delete(long id) {
+		try {
+			deleteStmt.setLong(1,id);
+			int n = deleteStmt.executeUpdate();
+			if( n==0 ) {
+				logger.error("delete not found for id="+id);
+			}
+		} catch(SQLException e) {
+			logger.error("update failed",e);
+		}
+	}
+
+	void begin() {
+		try {
+			if( trans++ == 0 )
+				con.setAutoCommit(false);
+		} catch(SQLException e) {
+			logger.error("begin failed",e);
+		}
+	}
+
+	void commit() {
+		try {
+			if( trans <= 0 ) {
+				logger.error("commit not in transaction");
+				return;
+			}
+			if( --trans == 0 )
+				con.setAutoCommit(true);
+		} catch(SQLException e) {
+			logger.error("begin failed",e);
+		}
+	}
+
+	void rollback() {
+		try {
+			if( --trans != 0 ) {
+				logger.error("rollback failed trans="+trans);
+				return;
+			}
+			con.rollback();
+			con.setAutoCommit(true);
+		} catch(SQLException e) {
+			logger.error("begin failed",e);
+		}
+	}
+
+}