Mercurial Hosting > luan
changeset 1387:bc40bc9aab3a
start postgres backup
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Mon, 02 Sep 2019 22:23:12 -0600 |
parents | dc36dd8bf839 |
children | 2024d23ddd64 |
files | examples/blog/src/delete.luan examples/blog/src/index.html.luan examples/blog/src/lib/Db.luan examples/blog/src/lib/Post.luan lib/postgresql-9.1-901.jdbc4.jar src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java |
diffstat | 8 files changed, 315 insertions(+), 8 deletions(-) [+] |
line wrap: on
line diff
diff -r dc36dd8bf839 -r bc40bc9aab3a examples/blog/src/delete.luan --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/examples/blog/src/delete.luan Mon Sep 02 22:23:12 2019 -0600 @@ -0,0 +1,13 @@ +local Luan = require "luan:Luan.luan" +local error = Luan.error +local String = require "luan:String.luan" +local to_number = String.to_number or error() +local Http = require "luan:http/Http.luan" +local Post = require "site:/lib/Post.luan" + + +return function() + local post_id = to_number(Http.request.parameters.post) or error() + Post.delete_by_id(post_id) + Http.response.send_redirect("/") +end
diff -r dc36dd8bf839 -r bc40bc9aab3a examples/blog/src/index.html.luan --- a/examples/blog/src/index.html.luan Thu Aug 22 12:32:20 2019 -0600 +++ b/examples/blog/src/index.html.luan Mon Sep 02 22:23:12 2019 -0600 @@ -39,7 +39,11 @@ %> <a name="p<%= post.id %>"> <h2><%= post.subject %></h2> - <p><%= Time.format(post.date) %> - <a href="edit.html?post=<%= post.id %>">Edit</a></p> + <p> + <%= Time.format(post.date) %> + - <a href="edit.html?post=<%= post.id %>">Edit</a> + - <a href="delete?post=<%= post.id %>">Delete</a> + </p> <pre><%= bbcode_to_html(html_encode(post.content)) %></pre> <hr> <%
diff -r dc36dd8bf839 -r bc40bc9aab3a examples/blog/src/lib/Db.luan --- a/examples/blog/src/lib/Db.luan Thu Aug 22 12:32:20 2019 -0600 +++ b/examples/blog/src/lib/Db.luan Mon Sep 02 22:23:12 2019 -0600 @@ -4,9 +4,13 @@ local Db = {} +local function completer(doc) + return doc +end + function Db.new(lucene_dir) local dir = Io.uri(lucene_dir) - local db = Lucene.index( dir, Lucene.type.english, {"subject","content"} ) + local db = Lucene.index( dir, Lucene.type.english, {"subject","content"}, completer ) -- this is how you index a field -- db.indexed_fields.post_date = Lucene.type.long
diff -r dc36dd8bf839 -r bc40bc9aab3a examples/blog/src/lib/Post.luan --- a/examples/blog/src/lib/Post.luan Thu Aug 22 12:32:20 2019 -0600 +++ b/examples/blog/src/lib/Post.luan Mon Sep 02 22:23:12 2019 -0600 @@ -67,4 +67,8 @@ return posts end +function Post.delete_by_id(id) + db.delete("id:"..id) +end + return Post
diff -r dc36dd8bf839 -r bc40bc9aab3a lib/postgresql-9.1-901.jdbc4.jar Binary file lib/postgresql-9.1-901.jdbc4.jar has changed
diff -r dc36dd8bf839 -r bc40bc9aab3a src/luan/modules/lucene/Lucene.luan --- a/src/luan/modules/lucene/Lucene.luan Thu Aug 22 12:32:20 2019 -0600 +++ b/src/luan/modules/lucene/Lucene.luan Mon Sep 02 22:23:12 2019 -0600 @@ -35,12 +35,12 @@ Lucene.literal = SaneQueryParser.literal -function Lucene.index(index_dir,default_type,default_fields) +function Lucene.index(index_dir,default_type,default_fields,postgres_backup) type(index_dir)=="table" or error "index_dir must be table" index_dir.to_uri_string and matches(index_dir.to_uri_string(),"^file:") or error "must be file" local index = {} index.dir = index_dir - local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,default_type,default_fields) + local java_index, closer = LuceneIndex.getLuceneIndex(index_dir.java.file,default_type,default_fields,postgres_backup) index.java = java_index index.closer = closer or error() @@ -69,6 +69,9 @@ index.count_tokens = java_index.count_tokens index.close = closer.close + index.has_postgres_backup = java_index.hasPostgresBackup() + index.rebuild_postgres_backup = java_index.rebuild_postgres_backup + function index.search( query, from, to, options ) from or error "missing 'from' parameter" to or error "missing 'to' parameter"
diff -r dc36dd8bf839 -r bc40bc9aab3a src/luan/modules/lucene/LuceneIndex.java --- a/src/luan/modules/lucene/LuceneIndex.java Thu Aug 22 12:32:20 2019 -0600 +++ b/src/luan/modules/lucene/LuceneIndex.java Mon Sep 02 22:23:12 2019 -0600 @@ -45,6 +45,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.apache.lucene.search.Query; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.Sort; @@ -110,14 +111,14 @@ private static Map<String,LuceneIndex> indexes = new HashMap<String,LuceneIndex>(); - public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields) + public static Object[] getLuceneIndex(Luan luan,File indexDir,FieldParser defaultFieldParser,String[] defaultFields,LuanFunction completer) throws LuanException, IOException { String key = indexDir.getCanonicalPath(); synchronized(indexes) { LuceneIndex li = indexes.get(key); if( li == null ) { - li = new LuceneIndex(indexDir,defaultFieldParser,defaultFields,key); + li = new LuceneIndex(indexDir,defaultFieldParser,defaultFields,key,completer); li.openCount = 1; indexes.put(key,li); } else { @@ -157,7 +158,9 @@ private final FieldParser defaultFieldParser; private final String[] defaultFields; - private LuceneIndex(File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key) + private final PostgresBackup postgresBackup; + + private LuceneIndex(File indexDir,FieldParser defaultFieldParser,String[] defaultFields,String key,LuanFunction completer) throws LuanException, IOException { this.key = key; @@ -174,6 +177,9 @@ } this.analyzer = analyzer; reopen(); + postgresBackup = completer!=null ? PostgresBackup.newInstance() : null; + if( postgresBackup != null && postgresBackup.wasCreated ) + rebuild_postgres_backup(completer); } public void reopen() throws LuanException, IOException { @@ -199,6 +205,8 @@ try { writer.deleteAll(); id = idLim = 0; + if( postgresBackup != null ) + postgresBackup.deleteAll(); if(commit) writer.commit(); } finally { wrote(); @@ -212,6 +220,28 @@ return new Term(key,br); } + private void backupDelete(Query query) + throws IOException + { + if( postgresBackup != null ) { + final List<Long> ids = new ArrayList<Long>(); + IndexSearcher searcher = openSearcher(); + MyCollector col = new MyCollector() { + @Override public void collect(int iDoc) throws IOException { + Document doc = searcher.doc( docBase + iDoc ); + Long id = (Long)doc.getField("id").numericValue(); + ids.add(id); + } + }; + searcher.search(query,col); + postgresBackup.begin(); + for( Long id : ids ) { + postgresBackup.delete(id); + } + postgresBackup.commit(); + } + } + public void delete(String queryStr) throws IOException, ParseException { @@ -220,6 +250,7 @@ boolean commit = !writeLock.isHeldByCurrentThread(); writeLock.lock(); try { + backupDelete(query); writer.deleteDocuments(query); if(commit) writer.commit(); } finally { @@ -235,6 +266,9 @@ public void save(LuanTable doc,LuanTable boosts) throws LuanException, IOException { + if( boosts!=null && postgresBackup!=null ) + logger.error("boosts are not saved to postgres backup"); + Object obj = doc.get("id"); Long id; try { @@ -250,8 +284,12 @@ id = nextId(); doc.put("id",id); writer.addDocument(toLucene(doc,boosts)); + if( postgresBackup != null ) + postgresBackup.add(id,doc); } else { writer.updateDocument( term("id",id), toLucene(doc,boosts) ); + if( postgresBackup != null ) + postgresBackup.update(id,doc); } if(commit) writer.commit(); } finally { @@ -263,16 +301,31 @@ public Object run_in_transaction(LuanFunction fn) throws IOException, LuanException { boolean commit = !writeLock.isHeldByCurrentThread(); writeLock.lock(); + boolean ok = false; try { + if( commit && postgresBackup != null ) + postgresBackup.begin(); Object rtn = fn.call(); - if(commit) writer.commit(); + ok = true; + if(commit) { + if( postgresBackup != null ) + postgresBackup.commit(); + writer.commit(); + } return rtn; } finally { + if( !ok && commit ) { + if( postgresBackup != null ) + postgresBackup.rollback(); + writer.rollback(); + reopen(); + } wrote(); writeLock.unlock(); } } + // ??? public Object run_in_lock(LuanFunction fn) throws IOException, LuanException { if( writeLock.isHeldByCurrentThread() ) throw new RuntimeException(); @@ -373,6 +426,8 @@ } public void doClose() throws IOException { + if( postgresBackup != null ) + postgresBackup.close(); writer.close(); reader.close(); } @@ -725,4 +780,48 @@ return n; } + + + public boolean hasPostgresBackup() { + return postgresBackup != null; + } + + public void rebuild_postgres_backup(LuanFunction completer) + throws IOException, LuanException + { + writeLock.lock(); + boolean ok = false; + try { + postgresBackup.begin(); + postgresBackup.deleteAll(); + Query query = new PrefixQuery(new Term("id")); + IndexSearcher searcher = openSearcher(); + MyCollector col = new MyCollector() { + @Override public void collect(int iDoc) throws IOException { + try { + Document doc = searcher.doc( docBase + iDoc ); + LuanTable tbl = toTable(completer.luan(),doc); + tbl = (LuanTable)completer.call(tbl); + Long id = (Long)tbl.get("id"); + //logger.info("id = "+id); + postgresBackup.add(id,tbl); + } catch(LuanException e) { + throw new LuanRuntimeException(e); + } + } + }; + try { + searcher.search(query,col); + } catch(LuanRuntimeException e) { + throw (LuanException)e.getCause(); + } + ok = true; + postgresBackup.commit(); + } finally { + if( !ok ) + postgresBackup.rollback(); + writeLock.unlock(); + } + } + }
diff -r dc36dd8bf839 -r bc40bc9aab3a src/luan/modules/lucene/PostgresBackup.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/modules/lucene/PostgresBackup.java Mon Sep 02 22:23:12 2019 -0600 @@ -0,0 +1,180 @@ +package luan.modules.lucene; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.Properties; +import luan.LuanTable; +import luan.LuanException; +import luan.modules.parsers.LuanToString; +import luan.lib.logging.Logger; +import luan.lib.logging.LoggerFactory; + + +final class PostgresBackup { + private static final Logger logger = LoggerFactory.getLogger(PostgresBackup.class); + + static PostgresBackup newInstance() { + try { + return new PostgresBackup(); + } catch(ClassNotFoundException e) { + logger.error("creation failed",e); + return null; + } catch(SQLException e) { + logger.error("creation failed",e); + return null; + } + } + + final boolean wasCreated; + private final Connection con; + private final PreparedStatement insertStmt; + private final PreparedStatement updateStmt; + private final PreparedStatement deleteStmt; + private int trans = 0; + + private PostgresBackup() + throws ClassNotFoundException, SQLException + { + Class.forName("org.postgresql.Driver"); + + String url = "jdbc:postgresql://localhost:5432/luan"; + Properties props = new Properties(); + props.setProperty("user","postgres"); + props.setProperty("password",""); + + con = DriverManager.getConnection(url,props); + + Statement stmt = con.createStatement(); + boolean hasTable = stmt.executeQuery( + "select * from information_schema.tables where table_name='lucene'" + ).next(); + if( !hasTable ) { + stmt.executeUpdate( + "create table lucene (" + +" id integer not null primary key," + +" data text not null" + +")" + ); + } + stmt.close(); + wasCreated = !hasTable; + + insertStmt = con.prepareStatement( + "insert into lucene (id,data) values (?,?)" + ); + updateStmt = con.prepareStatement( + "update lucene set data=? where id=?" + ); + deleteStmt = con.prepareStatement( + "delete from lucene where id=?" + ); + } + + void close() { + try { + insertStmt.close(); + updateStmt.close(); + deleteStmt.close(); + con.close(); + } catch(SQLException e) { + logger.error("close failed",e); + } + } + + protected void finalize() throws Throwable { + super.finalize(); + if( !con.isClosed() ) { + logger.error("con not closed"); + con.close(); + } + } + + void add(long id,LuanTable doc) throws LuanException { + try { +//logger.info("getAutoCommit="+con.getAutoCommit()); + String data = LuanToString.toString(doc,true); + insertStmt.setLong(1,id); + insertStmt.setString(2,data); + insertStmt.executeUpdate(); + } catch(SQLException e) { + logger.error("add failed",e); + } + } + + void update(long id,LuanTable doc) throws LuanException { + try { + String data = LuanToString.toString(doc,true); + updateStmt.setString(1,data); + updateStmt.setLong(2,id); + int n = updateStmt.executeUpdate(); + if( n==0 ) { + logger.error("update not found for id="+id+", trying add"); + add(id,doc); + } else if( n!=1 ) + throw new RuntimeException(); + } catch(SQLException e) { + logger.error("update failed",e); + } + } + + void deleteAll() { + try { + Statement stmt = con.createStatement(); + stmt.executeUpdate("delete from lucene"); + stmt.close(); + } catch(SQLException e) { + logger.error("update failed",e); + } + } + + void delete(long id) { + try { + deleteStmt.setLong(1,id); + int n = deleteStmt.executeUpdate(); + if( n==0 ) { + logger.error("delete not found for id="+id); + } + } catch(SQLException e) { + logger.error("update failed",e); + } + } + + void begin() { + try { + if( trans++ == 0 ) + con.setAutoCommit(false); + } catch(SQLException e) { + logger.error("begin failed",e); + } + } + + void commit() { + try { + if( trans <= 0 ) { + logger.error("commit not in transaction"); + return; + } + if( --trans == 0 ) + con.setAutoCommit(true); + } catch(SQLException e) { + logger.error("begin failed",e); + } + } + + void rollback() { + try { + if( --trans != 0 ) { + logger.error("rollback failed trans="+trans); + return; + } + con.rollback(); + con.setAutoCommit(true); + } catch(SQLException e) { + logger.error("begin failed",e); + } + } + +}