Mercurial Hosting > luan

changeset 1453:928be2a4d565

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
fix postgres backup
author Franklin Schmidt <fschmidt@gmail.com>
date Mon, 02 Mar 2020 15:09:10 -0700
parents c15444f4da05
children 219f2b937f2b
files src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/PostgresBackup.java
diffstat 3 files changed, 63 insertions(+), 47 deletions(-) [+]
line wrap: on
line diff
--- a/src/luan/modules/lucene/Lucene.luan	Sun Mar 01 16:05:26 2020 -0700
+++ b/src/luan/modules/lucene/Lucene.luan	Mon Mar 02 15:09:10 2020 -0700
@@ -39,9 +39,10 @@
 	options = options or {}
 	local index = {}
 	index.dir = index_dir
+	index.completer = options.completer
+	options.completer = nil
 	local java_index = LuceneIndex.getLuceneIndex(index_dir.java.file,options)
 	index.java = java_index
-	index.completer = options.completer
 
 	index.indexed_fields = {}
 	local mt = {}
@@ -59,7 +60,7 @@
 	index.search_in_transaction = java_index.search_in_transaction
 	index.delete_all = java_index.delete_all
 	index.delete = java_index.delete
-	index.save = java_index.save
+	--index.save = java_index.save
 	index.run_in_transaction = java_index.run_in_transaction
 	index.ensure_open = java_index.ensure_open
 	index.next_id = java_index.nextId
@@ -68,15 +69,20 @@
 	index.count_tokens = java_index.count_tokens
 	--index.close = java_index.close
 
-	index.has_postgres_backup = java_index.hasPostgresBackup()
-	function index.rebuild_postgres_backup()
-		java_index.rebuild_postgres_backup(index.completer)
+	local java_save = java_index.save
+	function index.save(doc,boosts)
+		java_save(index.completer,doc,boosts)
 	end
-	index.restore_from_postgres = java_index.restore_from_postgres
-	index.force_restore_from_postgres = java_index.force_restore_from_postgres
-	function index.check()
-		java_index.check(index.completer)
+
+	index.has_postgres_backup = java_index.hasPostgresBackup()
+	index.rebuild_postgres_backup = java_index.rebuild_postgres_backup
+	function index.restore_from_postgres()
+		java_index.restore_from_postgres(index.completer)
 	end
+	function index.force_restore_from_postgres()
+		java_index.force_restore_from_postgres(index.completer)
+	end
+	index.check = java_index.check
 
 	function index.search( query, from, to, options )
 		from or error "missing 'from' parameter"
--- a/src/luan/modules/lucene/LuceneIndex.java	Sun Mar 01 16:05:26 2020 -0700
+++ b/src/luan/modules/lucene/LuceneIndex.java	Mon Mar 02 15:09:10 2020 -0700
@@ -134,8 +134,8 @@
 	private AtomicInteger writeCounter = new AtomicInteger();
 
 	private Set<String> indexOnly = new HashSet<String>();
-	private final FieldParser defaultFieldParser;
-	private final String[] defaultFields;
+//	private final FieldParser defaultFieldParser;
+//	private final String[] defaultFields;
 
 	private final PostgresBackup postgresBackup;
 	private boolean wasCreated;
@@ -148,13 +148,12 @@
 		FieldParser defaultFieldParser = (FieldParser)options.remove("default_type");
 		LuanTable defaultFieldsTbl = Utils.removeTable(options,"default_fields");
 		String[] defaultFields = defaultFieldsTbl==null ? null : (String[])defaultFieldsTbl.asList().toArray(new String[0]);
-		LuanFunction completer = Utils.removeFunction(options,"completer");
 		LuanTable postgresSpec = Utils.removeTable(options,"postgres_spec");
 		Utils.checkEmpty(options);
 
 		this.luanLogger = luan.getLogger(LuceneIndex.class);
-		this.defaultFieldParser = defaultFieldParser;
-		this.defaultFields = defaultFields;
+//		this.defaultFieldParser = defaultFieldParser;
+//		this.defaultFields = defaultFields;
 		mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
 		mfp.fields.put( "type", STRING_FIELD_PARSER );
 		mfp.fields.put( "id", NumberFieldParser.LONG );
@@ -169,12 +168,10 @@
 		if( postgresSpec == null ) {
 			postgresBackup = null;
 		} else {
-			if( completer == null )
-				throw new LuanException("completer is required for postgres_spec");
 			postgresBackup = new PostgresBackup(luan,postgresSpec);
 			if( !wasCreated && postgresBackup.wasCreated ) {
 				luanLogger.error("rebuilding postgres backup");
-				rebuild_postgres_backup(completer);
+				rebuild_postgres_backup(luan);
 /*
 			} else if( wasCreated && !postgresBackup.wasCreated ) {
 				luanLogger.error("restoring from postgres");
@@ -270,7 +267,7 @@
 		indexOnly.addAll(fields);
 	}
 
-	public void save(LuanTable doc,LuanTable boosts)
+	public void save(LuanFunction completer,LuanTable doc,LuanTable boosts)
 		throws LuanException, IOException, SQLException
 	{
 		if( boosts!=null && postgresBackup!=null )
@@ -292,11 +289,11 @@
 				doc.put("id",id);
 				if( postgresBackup != null )
 					postgresBackup.add(doc);
-				writer.addDocument(toLucene(doc,boosts));
+				writer.addDocument(toLucene(completer,doc,boosts));
 			} else {
 				if( postgresBackup != null )
 					postgresBackup.update(doc);
-				writer.updateDocument( term("id",id), toLucene(doc,boosts) );
+				writer.updateDocument( term("id",id), toLucene(completer,doc,boosts) );
 			}
 			if(commit) writer.commit();
 		} finally {
@@ -648,7 +645,9 @@
 			throw new LuanException("invalid value type "+value.getClass()+"' for '"+name+"'");
 	}
 
-	private Document toLucene(LuanTable table,LuanTable boosts) throws LuanException {
+	private Document toLucene(LuanFunction completer,LuanTable table,LuanTable boosts) throws LuanException {
+		if( completer != null )
+			table = (LuanTable)completer.call(table);
 		return toLucene(table.iterable(),boosts);
 	}
 
@@ -804,7 +803,7 @@
 		return postgresBackup != null;
 	}
 
-	public void rebuild_postgres_backup(LuanFunction completer)
+	public void rebuild_postgres_backup(Luan luan)
 		throws IOException, LuanException, SQLException
 	{
 		luanLogger.info("start rebuild_postgres_backup");
@@ -819,8 +818,7 @@
 				@Override public void collect(int iDoc) throws IOException {
 					try {
 						Document doc = searcher.doc( docBase + iDoc );
-						LuanTable tbl = toTable(completer.luan(),doc);
-						tbl = (LuanTable)completer.call(tbl);
+						LuanTable tbl = toTable(luan,doc);
 						postgresBackup.add(tbl);
 					} catch(LuanException e) {
 						throw new LuanRuntimeException(e);
@@ -845,16 +843,16 @@
 		luanLogger.info("end rebuild_postgres_backup");
 	}
 
-	public void restore_from_postgres()
+	public void restore_from_postgres(LuanFunction completer)
 		throws IOException, LuanException, SQLException, ParseException
 	{
 		if( postgresBackup!=null && wasCreated && !postgresBackup.wasCreated ) {
 			luanLogger.error("restoring from postgres");
-			force_restore_from_postgres();
+			force_restore_from_postgres(completer);
 		}
 	}
 
-	public void force_restore_from_postgres()
+	public void force_restore_from_postgres(LuanFunction completer)
 		throws IOException, LuanException, SQLException, ParseException
 	{
 		luanLogger.warn("start restore_from_postgres");
@@ -867,7 +865,7 @@
 		try {
 			writer.deleteAll();
 			long nextId = postgresBackup.maxId() + 1;
-			postgresBackup.restoreLucene(this);
+			postgresBackup.restoreLucene(this,completer);
 			id = idLim = nextId;
 			saveNextId(nextId);
 			ok = true;
@@ -884,13 +882,13 @@
 		luanLogger.warn("end restore_from_postgres");
 	}
 
-	void restore(LuanTable doc)
+	void restore(LuanFunction completer,LuanTable doc)
 		throws LuanException, IOException
 	{
-		writer.addDocument(toLucene(doc,null));
+		writer.addDocument(toLucene(completer,doc,null));
 	}
 
-	public void check(LuanFunction completer) throws IOException, SQLException, LuanException, ParseException {
+	public void check(Luan luan) throws IOException, SQLException, LuanException, ParseException {
 		boolean hasPostgres = postgresBackup != null;
 		String msg = "start check";
 		if( hasPostgres )
@@ -900,11 +898,11 @@
 		if( !status.clean )
 			luanLogger.error("index not clean");
 		if( hasPostgres )
-			checkPostgres(completer);
+			checkPostgres(luan);
 		luanLogger.info("end check");
 	}
 
-	private void checkPostgres(LuanFunction completer)
+	private void checkPostgres(Luan luan)
 		throws IOException, SQLException, LuanException, ParseException
 	{
 		//luanLogger.info("start postgres check");
@@ -935,19 +933,18 @@
 				long idPostgres = idsPostgres.get(iPostgres);
 				if( idLucene < idPostgres ) {
 					iLucene++;
-					checkPostgres(completer,postgresChecker,lts,idLucene);
+					checkPostgres(luan,postgresChecker,lts,idLucene);
 				} else if( idLucene > idPostgres ) {
 					iPostgres++;
-					checkPostgres(completer,postgresChecker,lts,idPostgres);
+					checkPostgres(luan,postgresChecker,lts,idPostgres);
 				} else {  // ==
 					LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
 					TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
 					if( td.totalHits != 1 )  throw new RuntimeException();
 					Document doc = searcher.doc( td.scoreDocs[0].doc );
-					LuanTable docLucene = toTable(completer.luan(),doc);
-					docLucene = (LuanTable)completer.call(docLucene);
+					LuanTable docLucene = toTable(luan,doc);
 					if( !equal(docPostgres,docLucene) ) {
-						checkPostgres(completer,postgresChecker,lts,idPostgres);
+						checkPostgres(luan,postgresChecker,lts,idPostgres);
 					}
 					iLucene++;
 					iPostgres++;
@@ -955,11 +952,11 @@
 			}
 			while( iLucene < nLucene ) {
 				long idLucene = idsLucene.get(iLucene++);
-				checkPostgres(completer,postgresChecker,lts,idLucene);
+				checkPostgres(luan,postgresChecker,lts,idLucene);
 			}
 			while( iPostgres < nPostgres ) {
 				long idPostgres = idsPostgres.get(iPostgres++);
-				checkPostgres(completer,postgresChecker,lts,idPostgres);
+				checkPostgres(luan,postgresChecker,lts,idPostgres);
 			}
 		} finally {
 			close(searcher);
@@ -967,7 +964,7 @@
 		}
 	}
 
-	private void checkPostgres(LuanFunction completer,PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
+	private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
 		throws IOException, SQLException, LuanException, ParseException
 	{
 		//luanLogger.info("check id "+id);
@@ -982,8 +979,7 @@
 					docLucene = null;
 				} else if( td.totalHits == 1 ) {
 					Document doc = searcher.doc( td.scoreDocs[0].doc );
-					docLucene = toTable(completer.luan(),doc);
-					docLucene = (LuanTable)completer.call(docLucene);
+					docLucene = toTable(luan,doc);
 				} else
 					throw new RuntimeException();
 				if( docPostgres == null ) {
@@ -1008,8 +1004,22 @@
 		}
 	}
 
-	private boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
-		return t1!=null && t2!=null && t1.asMap().equals(t2.asMap());
+	private static boolean equal(LuanTable t1,LuanTable t2) throws LuanException {
+		return t1!=null && t2!=null && toJava(t1).equals(toJava(t2));
 	}
 
+	private static Map toJava(LuanTable t) throws LuanException {
+		Map map = t.asMap();
+		for( Object obj : map.entrySet() ) {
+			Map.Entry entry = (Map.Entry)obj;
+			Object value = entry.getValue();
+			if( value instanceof LuanTable ) {
+				LuanTable v = (LuanTable)value;
+				if( !v.isList() )
+					sysLogger.error("not list");
+				entry.setValue(v.asList());
+			}
+		}
+		return map;
+	}
 }
--- a/src/luan/modules/lucene/PostgresBackup.java	Sun Mar 01 16:05:26 2020 -0700
+++ b/src/luan/modules/lucene/PostgresBackup.java	Mon Mar 02 15:09:10 2020 -0700
@@ -158,7 +158,7 @@
 		con.setAutoCommit(true);
 	}
 
-	void restoreLucene(LuceneIndex li)
+	void restoreLucene(LuceneIndex li,LuanFunction completer)
 		throws LuanException, IOException, SQLException, ParseException
 	{
 		Luan luan = new Luan();
@@ -167,7 +167,7 @@
 		while( rs.next() ) {
 			String data = rs.getString("data");
 			LuanTable doc = (LuanTable)LuanParser.parse(luan,data);
-			li.restore(doc);
+			li.restore(completer,doc);
 		}
 		stmt.close();
 	}