Mercurial Hosting > luan
changeset 1696:2958cf04d844
remove postgres backup
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 26 Jun 2022 14:40:08 -0600 |
parents | 25833dd89844 |
children | aff2309ae510 |
files | examples/blog/src/lib/Db.luan src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java |
diffstat | 4 files changed, 10 insertions(+), 515 deletions(-) [+] |
line wrap: on
line diff
diff -r 25833dd89844 -r 2958cf04d844 examples/blog/src/lib/Db.luan --- a/examples/blog/src/lib/Db.luan Mon Jun 20 16:37:23 2022 -0600 +++ b/examples/blog/src/lib/Db.luan Sun Jun 26 14:40:08 2022 -0600 @@ -17,21 +17,16 @@ BackupIndexWriter.backupDomains = {"localhost"} end ---local postgres_spec = Hosted.postgres_spec() ---logger.info(stringify(postgres_spec)) - local dir = uri("site:/private/local/lucene") local Db = Lucene.index( dir, { default_type = Lucene.type.english default_fields = {"subject","content"} - --postgres_spec = postgres_spec log_dir = uri("site:/private/local/lucene_log") } ) -- this is how you index a field -- db.indexed_fields.post_date = Lucene.type.long ---Db.restore_from_postgres() Db.restore_from_log() Db.update{ [1] = function()
diff -r 25833dd89844 -r 2958cf04d844 src/luan/modules/lucene/Lucene.luan --- a/src/luan/modules/lucene/Lucene.luan Mon Jun 20 16:37:23 2022 -0600 +++ b/src/luan/modules/lucene/Lucene.luan Sun Jun 26 14:40:08 2022 -0600 @@ -113,10 +113,6 @@ index.restore_from_log = java_index.restore_from_log index.force_restore_from_log = java_index.force_restore_from_log - index.has_postgres_backup = java_index.hasPostgresBackup() - index.rebuild_postgres_backup = java_index.rebuild_postgres_backup - index.restore_from_postgres = java_index.restore_from_postgres - index.force_restore_from_postgres = java_index.force_restore_from_postgres index.check = java_index.check function index.not_in_transaction() end
diff -r 25833dd89844 -r 2958cf04d844 src/luan/modules/lucene/LuceneIndex.java --- a/src/luan/modules/lucene/LuceneIndex.java Mon Jun 20 16:37:23 2022 -0600 +++ b/src/luan/modules/lucene/LuceneIndex.java Sun Jun 26 14:40:08 2022 -0600 @@ -7,7 +7,6 @@ import java.io.IOException; import java.lang.ref.Reference; import java.lang.ref.WeakReference; -import java.sql.SQLException; import java.util.Arrays; import java.util.Iterator; import java.util.Map; @@ -100,7 +99,7 @@ private static Map<String,Reference<LuceneIndex>> indexes = new HashMap<String,Reference<LuceneIndex>>(); public static LuceneIndex getLuceneIndex(Luan luan,File indexDir,LuanTable options) - throws LuanException, IOException, ClassNotFoundException, SQLException + throws LuanException, IOException, ClassNotFoundException { String key = indexDir.getCanonicalPath(); synchronized(indexes) { @@ -143,7 +142,6 @@ private AtomicInteger writeCounter = new AtomicInteger(); private final GoodIndexWriterConfig config; - private final PostgresBackup postgresBackup; private boolean wasCreated; private final File logDir; private final long logTime; @@ -151,14 +149,13 @@ private final String domain; private LuceneIndex(Luan luan,File indexDir,LuanTable options) - throws LuanException, IOException, ClassNotFoundException, SQLException + throws LuanException, IOException, ClassNotFoundException { options = new LuanTable(options); this.version = options.remove("version"); FieldParser defaultFieldParser = (FieldParser)options.remove("default_type"); LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields"); String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]); - LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec"); LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); logDir = (File)options.remove("log_dir"); logTime = (Long)options.remove("log_time"); @@ -185,20 +182,6 @@ this.analyzer = analyzer; this.config = new SupplementingConfig(luceneVersion,mfp,supplementer); wasCreated = reopen(); - if( postgresSpec == null ) { - postgresBackup = null; - } else { - postgresBackup = new PostgresBackup(postgresSpec); - if( !wasCreated && postgresBackup.wasCreated ) { - logger.error("rebuilding postgres backup"); - rebuild_postgres_backup(luan); -/* - } else if( wasCreated && !postgresBackup.wasCreated ) { - logger.error("restoring from postgres"); - restore_from_postgres(); -*/ - } - } } public boolean reopen() throws IOException { @@ -222,14 +205,12 @@ writeCounter.incrementAndGet(); } - public void delete_all() throws IOException, SQLException { + public void delete_all() throws IOException { boolean commit = !writeLock.isHeldByCurrentThread(); writeLock.lock(); try { writer.deleteAll(); id = 0; - if( postgresBackup != null ) - postgresBackup.deleteAll(); if(commit) writer.commit(); } finally { wrote(); @@ -243,43 +224,14 @@ return new Term(key,br); } - private static final Set<String> ID_SET = Collections.singleton("id"); - - private void backupDelete(Query query) - throws IOException, SQLException, LuanException - { - if( postgresBackup != null ) { - final List<Long> ids = new ArrayList<Long>(); - IndexSearcher searcher = openSearcher(); - try { - MyCollector col = new MyCollector() { - @Override public void collect(int iDoc) throws IOException { - Document doc = searcher.doc( docBase + iDoc, ID_SET ); - Long id = (Long)doc.getField("id").numericValue(); - ids.add(id); - } - }; - searcher.search(query,col); - } finally { - close(searcher); - } - postgresBackup.begin(); - for( Long id : ids ) { - postgresBackup.delete(id); - } - postgresBackup.commit(); - } - } - public void delete(String queryStr) - throws IOException, ParseException, SQLException, LuanException + throws IOException, ParseException, LuanException { Query query = GoodQueryParser.parseQuery(mfp,queryStr); boolean commit = !writeLock.isHeldByCurrentThread(); writeLock.lock(); try { - backupDelete(query); writer.deleteDocuments(query); if(commit) writer.commit(); } finally { @@ -305,7 +257,7 @@ } public void save( Luan luan, LuanTable doc, LuanTable unstored, Map<String,Float> boosts ) - throws LuanException, IOException, SQLException + throws LuanException, IOException { Object obj = doc.get(luan,"id"); Long id; @@ -325,8 +277,6 @@ throw new LuanException("boosts required with unstored"); if( id != null ) throw new LuanException("update not supported"); - if( postgresBackup != null ) - throw new LuanException("not supported with postgres backup"); if( !(writer instanceof LuceneIndexWriter) ) throw new LuanException("not supported with index logging"); id = ++this.id; @@ -336,12 +286,8 @@ } else if( id == null ) { id = ++this.id; doc.put(luan,"id",id); - if( postgresBackup != null ) - postgresBackup.add(luan,doc); writer.addDocument(toLucene(doc)); } else { - if( postgresBackup != null ) - postgresBackup.update(luan,doc); writer.updateDocument( "id", toLucene(doc) ); } if(commit) writer.commit(); @@ -356,26 +302,20 @@ } public Object run_in_transaction(Luan luan,LuanFunction fn) - throws IOException, LuanException, SQLException + throws IOException, LuanException { boolean commit = !writeLock.isHeldByCurrentThread(); writeLock.lock(); boolean ok = false; try { - if( commit && postgresBackup != null ) - postgresBackup.begin(); Object rtn = fn.call(luan); ok = true; if(commit) { - if( postgresBackup != null ) - postgresBackup.commit(); writer.commit(); } return rtn; } finally { if( !ok && commit ) { - if( postgresBackup != null ) - postgresBackup.rollback(); writer.rollback(); reopen(); } @@ -474,17 +414,15 @@ super.finalize(); } - public void close() throws IOException, SQLException { + public void close() throws IOException { closeWriter(); reader.close(); } - private void closeWriter() throws IOException, SQLException { + private void closeWriter() throws IOException { writeLock.lock(); try { writer.close(); - if( postgresBackup != null ) - postgresBackup.close(); } finally { writeLock.unlock(); } @@ -717,91 +655,6 @@ } - - public boolean hasPostgresBackup() { - return postgresBackup != null; - } - - public void rebuild_postgres_backup(Luan luan) - throws IOException, LuanException, SQLException - { - logger.info("start rebuild_postgres_backup"); - writeLock.lock(); - IndexSearcher searcher = openSearcher(); - boolean ok = false; - try { - postgresBackup.begin(); - postgresBackup.deleteAll(); - Query query = new PrefixQuery(new Term("id")); - MyCollector col = new MyCollector() { - @Override public void collect(int iDoc) throws IOException { - try { - Document doc = searcher.doc( docBase + iDoc ); - LuanTable tbl = toTable(doc); - postgresBackup.add(luan,tbl); - } catch(LuanException e) { - throw new LuanRuntimeException(e); - } catch(SQLException e) { - throw new RuntimeException(e); - } - } - }; - try { - searcher.search(query,col); - } catch(LuanRuntimeException e) { - throw (LuanException)e.getCause(); - } - ok = true; - postgresBackup.commit(); - } finally { - close(searcher); - if( !ok ) - postgresBackup.rollback(); - writeLock.unlock(); - } - logger.info("end rebuild_postgres_backup"); - } - - public void restore_from_postgres() - throws IOException, LuanException, SQLException, ParseException - { - if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) { - logger.error("restoring from postgres"); - force_restore_from_postgres(); - } - } - - public void force_restore_from_postgres() - throws IOException, LuanException, SQLException, ParseException - { - logger.warn("start restore_from_postgres"); - if( postgresBackup==null ) - throw new NullPointerException(); - if( writeLock.isHeldByCurrentThread() ) - throw new RuntimeException(); - writeLock.lock(); - boolean ok = false; - try { - writer.tag("restore_from_postgres"); - writer.deleteAll(); - postgresBackup.restoreLucene(this); - ok = true; - writer.commit(); - wrote(); - ensure_open(); // refresh searcher - initId(); - wasCreated = false; - } finally { - if( !ok ) { - writer.rollback(); - reopen(); - } - wrote(); - writeLock.unlock(); - } - logger.warn("end restore_from_postgres"); - } - void restore(LuanTable doc) throws LuanException, IOException { @@ -823,7 +676,7 @@ } public void restore_from_log(Luan luan,LuanFunction handler) - throws IOException, LuanException, SQLException, ParseException + throws IOException, LuanException, ParseException { LoggingIndexWriter loggingWriter = (LoggingIndexWriter)writer; if( wasCreated && !loggingWriter.wasCreated ) { @@ -860,11 +713,8 @@ logger.warn("end force_restore_from_log"); } - public void check() throws IOException, SQLException, LuanException, ParseException { - boolean hasPostgres = postgresBackup != null; + public void check() throws IOException, LuanException, ParseException { String msg = "start check"; - if( hasPostgres ) - msg += " with postgres"; logger.info(msg); CheckIndex.Status status = new CheckIndex(fsDir).checkIndex(); if( !status.clean ) @@ -874,137 +724,7 @@ logger.info("log check"); boolean ok = loggingWriter.check(ID_SORT); } - if( hasPostgres ) { - logger.info("postgres check"); - checkPostgres(); - } logger.info("end check"); } - private void checkPostgres() - throws IOException, SQLException, LuanException, ParseException - { - final PostgresBackup.Checker postgresChecker = postgresBackup.newChecker(); - final IndexSearcher searcher = openSearcher(); - try { - final List<Long> idsLucene = new ArrayList<Long>(); - Query query = new PrefixQuery(new Term("id")); - MyCollector col = new MyCollector() { - @Override public void collect(int iDoc) throws IOException { - Document doc = searcher.doc( docBase + iDoc ); - Long id = (Long)doc.getField("id").numericValue(); - idsLucene.add(id); - } - }; - searcher.search(query,col); - Collections.sort(idsLucene); - final List<Long> idsPostgres = postgresChecker.getIds(); - final int nLucene = idsLucene.size(); - final int nPostgres = idsPostgres.size(); - int iLucene = 0; - int iPostgres = 0; - LuanToString lts = new LuanToString(); - lts.settingsInit.strict = true; - lts.settingsInit.numberTypes = true; - while( iLucene < nLucene && iPostgres < nPostgres ) { - long idLucene = idsLucene.get(iLucene); - long idPostgres = idsPostgres.get(iPostgres); - if( idLucene < idPostgres ) { - iLucene++; - checkPostgres(postgresChecker,lts,idLucene); - } else if( idLucene > idPostgres ) { - iPostgres++; - checkPostgres(postgresChecker,lts,idPostgres); - } else { // == - LuanTable docPostgres = postgresChecker.getDoc(idPostgres); - TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); - if( td.totalHits != 1 ) throw new RuntimeException(); - Document doc = searcher.doc( td.scoreDocs[0].doc ); - LuanTable docLucene = toTable(doc); - if( !equal(docPostgres,docLucene) ) { - checkPostgres(postgresChecker,lts,idPostgres); - } - iLucene++; - iPostgres++; - } - } - while( iLucene < nLucene ) { - long idLucene = idsLucene.get(iLucene++); - checkPostgres(postgresChecker,lts,idLucene); - } - while( iPostgres < nPostgres ) { - long idPostgres = idsPostgres.get(iPostgres++); - checkPostgres(postgresChecker,lts,idPostgres); - } - } finally { - close(searcher); - postgresChecker.close(); - } - } - - private void checkPostgres(PostgresBackup.Checker postgresChecker,LuanToString lts,long id) - throws IOException, SQLException, LuanException, ParseException - { - //logger.info("check id "+id); - writeLock.lock(); - try { - final IndexSearcher searcher = openSearcher(); - try { - LuanTable docPostgres = postgresChecker.getDoc(id); - TopDocs td = searcher.search(new TermQuery(term("id",id)),1); - LuanTable docLucene; - if( td.totalHits == 0 ) { - docLucene = null; - } else if( td.totalHits == 1 ) { - Document doc = searcher.doc( td.scoreDocs[0].doc ); - docLucene = toTable(doc); - } else - throw new RuntimeException(); - if( docPostgres == null ) { - if( docLucene != null ) - logger.error("id "+id+" found in lucene but not postgres"); - return; - } - if( docLucene == null ) { - logger.error("id "+id+" found in postgres but not lucene"); - return; - } - if( !equal(docPostgres,docLucene) ) { - logger.error("id "+id+" not equal"); - logger.error("lucene = "+lts.toString(docLucene)); - logger.error("postgres = "+lts.toString(docPostgres)); - } - } finally { - close(searcher); - } - } finally { - writeLock.unlock(); - } - } - - private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException { - return t1!=null && t2!=null && toJava(t1).equals(toJava(t2)); - } - - private static Map toJava(LuanTable t) throws LuanException { - Map map = t.asMap(); - for( Iterator iter = map.entrySet().iterator(); iter.hasNext(); ) { - Map.Entry entry = (Map.Entry)iter.next(); - Object value = entry.getValue(); - if( value instanceof LuanTable ) { - LuanTable v = (LuanTable)value; - if( !v.isList() ) - logger.error("not list"); - List list = v.asList(); - if( list.isEmpty() ) { - iter.remove(); - } else if( list.size() == 1 ) { - entry.setValue(list.get(0)); - } else { - entry.setValue(list); - } - } - } - return map; - } }
diff -r 25833dd89844 -r 2958cf04d844 src/luan/modules/lucene/PostgresBackup.java --- a/src/luan/modules/lucene/PostgresBackup.java Mon Jun 20 16:37:23 2022 -0600 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,216 +0,0 @@ -package luan.modules.lucene; - -import java.io.IOException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.ResultSet; -import java.util.Properties; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import luan.Luan; -import luan.LuanTable; -import luan.LuanFunction; -import luan.LuanException; -import luan.modules.Utils; -import luan.modules.parsers.LuanToString; -import luan.modules.parsers.LuanParser; -import goodjava.parser.ParseException; -import goodjava.logging.Logger; -import goodjava.logging.LoggerFactory; - - -final class PostgresBackup { - private static final Logger logger = LoggerFactory.getLogger(PostgresBackup.class); - - final boolean wasCreated; - private final String url; - private final Properties props = new Properties(); - private final Connection con; - private final PreparedStatement insertStmt; - private final PreparedStatement updateStmt; - private final PreparedStatement deleteStmt; - private int trans = 0; - private final LuanToString luanToString; - - PostgresBackup(LuanTable spec) - throws ClassNotFoundException, SQLException, LuanException - { - spec = new LuanTable(spec); -/* - Class.forName("org.postgresql.Driver"); - url = "jdbc:postgresql://localhost:5432/luan"; - props.setProperty("user","postgres"); - props.setProperty("password",""); -*/ - String cls = "org.postgresql.Driver"; - if( !Utils.removeRequiredString(spec,"class").equals(cls) ) - throw new LuanException( "parameter 'class' must be '"+cls+"'" ); - Class.forName(cls); - url = Utils.removeRequiredString(spec,"url"); - props.setProperty( "user", Utils.removeRequiredString(spec,"user") ); - props.setProperty( "password", Utils.removeRequiredString(spec,"password") ); - Utils.checkEmpty(spec); - - con = newConnection(); - - Statement stmt = con.createStatement(); - boolean hasTable = stmt.executeQuery( - "select * from information_schema.tables where table_name='lucene'" - ).next(); - if( !hasTable ) { - stmt.executeUpdate( - "create table lucene (" - +" id integer not null primary key," - +" data text not null" - +")" - ); - } - stmt.close(); - wasCreated = !hasTable; - - insertStmt = con.prepareStatement( - "insert into lucene (id,data) values (?,?)" - ); - updateStmt = con.prepareStatement( - "update lucene set data=? where id=?" - ); - deleteStmt = con.prepareStatement( - "delete from lucene where id=?" - ); - - luanToString = new LuanToString(); - luanToString.settingsInit.strict = true; - luanToString.settingsInit.numberTypes = true; - } - - Connection newConnection() throws SQLException { - return DriverManager.getConnection(url,props); - } - - void close() throws SQLException { - insertStmt.close(); - updateStmt.close(); - deleteStmt.close(); - con.close(); - } - - protected void finalize() throws Throwable { - close(); - super.finalize(); - } - - void add(Luan luan,LuanTable doc) throws LuanException, SQLException { - Long id = (Long)doc.get(luan,"id"); - String data = luanToString.toString(doc); - insertStmt.setLong(1,id); - insertStmt.setString(2,data); - insertStmt.executeUpdate(); - } - - void update(Luan luan,LuanTable doc) throws LuanException, SQLException { - Long id = (Long)doc.get(luan,"id"); - String data = luanToString.toString(doc); - updateStmt.setString(1,data); - updateStmt.setLong(2,id); - int n = updateStmt.executeUpdate(); - if( n==0 ) { - logger.error("update not found for id="+id+", trying add"); - add(luan,doc); - } else if( n!=1 ) - throw new RuntimeException(); - } - - void deleteAll() throws SQLException { - Statement stmt = con.createStatement(); - stmt.executeUpdate("delete from lucene"); - stmt.close(); - } - - void delete(long id) throws SQLException, LuanException { - deleteStmt.setLong(1,id); - int n = deleteStmt.executeUpdate(); - if( n==0 ) - throw new LuanException("delete not found for id="+id); - } - - void begin() throws SQLException { - if( trans++ == 0 ) - con.setAutoCommit(false); - } - - void commit() throws SQLException, LuanException { - if( trans <= 0 ) - throw new LuanException("commit not in transaction"); - if( --trans == 0 ) - con.setAutoCommit(true); - } - - void rollback() throws SQLException, LuanException { - if( --trans != 0 ) - throw new LuanException("rollback failed trans="+trans); - con.rollback(); - con.setAutoCommit(true); - } - - void restoreLucene(LuceneIndex li) - throws LuanException, IOException, SQLException, ParseException - { - Statement stmt = con.createStatement(); - ResultSet rs = stmt.executeQuery("select data from lucene"); - while( rs.next() ) { - String data = rs.getString("data"); - LuanTable doc = (LuanTable)LuanParser.parse(data); - li.restore(doc); - } - stmt.close(); - } - - final class Checker { - private final Connection con; - private final PreparedStatement pstmt; - - Checker() throws SQLException { - con = newConnection(); - con.setReadOnly(true); - pstmt = con.prepareStatement( - "select data from lucene where id=?" - ); - } - - void close() throws SQLException { - pstmt.close(); - con.close(); - } - - List<Long> getIds() throws SQLException { - List<Long> ids = new ArrayList<Long>(); - Statement stmt = con.createStatement(); - ResultSet rs = stmt.executeQuery("select id from lucene order by id"); - while( rs.next() ) { - long id = rs.getLong("id"); - ids.add(id); - } - stmt.close(); - return ids; - } - - LuanTable getDoc(long id) throws SQLException, ParseException { - pstmt.setLong(1,id); - ResultSet rs = pstmt.executeQuery(); - if( !rs.next() ) - return null; - String data = rs.getString("data"); - LuanTable doc = (LuanTable)LuanParser.parse(data); - return doc; - } - } - - Checker newChecker() throws SQLException { - return new Checker(); - } - -}