Mercurial Hosting > luan

changeset 1672:8dd8c556c449

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
backup work
author Franklin Schmidt <fschmidt@gmail.com>
date Thu, 26 May 2022 21:10:02 -0600
parents 8066b8882732
children 1b9f9fdb3b41
files .hgignore backup/server.luan examples/blog/src/lib/Db.luan src/goodjava/lucene/backup/Backup.java src/goodjava/lucene/backup/BackupIndexWriter.java src/goodjava/lucene/backup/BackupServer.java src/luan/LuanJavaFunction.java src/luan/modules/JavaLuan.java src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java
diffstat 10 files changed, 133 insertions(+), 29 deletions(-) [+]
line wrap: on
line diff
diff -r 8066b8882732 -r 8dd8c556c449 .hgignore
--- a/.hgignore	Thu May 12 10:56:45 2022 -0600
+++ b/.hgignore	Thu May 26 21:10:02 2022 -0600
@@ -14,3 +14,5 @@
 host/logs/
 host/started.lock
 push.sh
+backup/logs/
+backup/backups/
diff -r 8066b8882732 -r 8dd8c556c449 backup/server.luan
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/backup/server.luan	Thu May 26 21:10:02 2022 -0600
@@ -0,0 +1,47 @@
+local Luan = require "luan:Luan.luan"
+local error = Luan.error
+local ipairs = Luan.ipairs or error()
+local Io = require "luan:Io.luan"
+
+require "java"
+local File = require "java:java.io.File"
+local BackupServer = require "java:goodjava.lucene.backup.BackupServer"
+local Logging = require "luan:logging/Logging.luan"
+
+local log_to_console = false
+for _, arg in ipairs{...} do
+	if arg == "console" then
+		log_to_console = true
+	end
+end
+if not log_to_console then
+	local FileWriter = require "java:java.io.FileWriter"
+	local LuanLogger = require "java:luan.modules.logging.LuanLogger"
+	local Layouts = require "java:goodjava.logger.Layouts"
+	local DateLayout = require "java:goodjava.logger.DateLayout"
+	local ListLayout = require "java:goodjava.logger.ListLayout"
+	local Level = require "java:goodjava.logger.Level"
+	local WriterAppender = require "java:goodjava.logger.WriterAppender"
+	local LevelAppender = require "java:goodjava.logger.LevelAppender"
+	local ListAppender = require "java:goodjava.logger.ListAppender"
+
+	local layout = ListLayout.new(DateLayout.new("yyyy-MM-dd HH:mm:ss,SSS")," ",Layouts.LEVEL_PADDED," ",Layouts.LOGGER," - ",Layouts.MESSAGE,"\n",Layouts.THROWABLE)
+
+	local function new_appender(file,level)
+		local writer = FileWriter.new(file,true)
+		local appender = WriterAppender.new(layout,writer)
+		appender = LevelAppender.new(appender,level)
+		return appender
+	end
+
+	Io.uri("file:logs").mkdir()
+	local err = new_appender("logs/error.log",Level.ERROR)
+	local warn = new_appender("logs/warn.log",Level.WARN)
+	local info = new_appender("logs/info.log",Level.INFO)
+	local appender = ListAppender.new(err,warn,info)
+	LuanLogger.configure(appender)
+end
+
+local dir = File.new("backups")
+local server = BackupServer.new(dir)
+server.start()
diff -r 8066b8882732 -r 8dd8c556c449 examples/blog/src/lib/Db.luan
--- a/examples/blog/src/lib/Db.luan	Thu May 12 10:56:45 2022 -0600
+++ b/examples/blog/src/lib/Db.luan	Thu May 26 21:10:02 2022 -0600
@@ -3,27 +3,36 @@
 local stringify = Luan.stringify or error()
 local Lucene = require "luan:lucene/Lucene.luan"
 local Io = require "luan:Io.luan"
-local Hosted = require "luan:host/Hosted.luan"
+local uri = Io.uri or error()
 local Time = require "luan:Time.luan"
 local Thread = require "luan:Thread.luan"
+local Hosted = require "luan:host/Hosted.luan"
 local Logging = require "luan:logging/Logging.luan"
 local logger = Logging.logger "Db"
 
 
-local postgres_spec = Hosted.postgres_spec()
+if not Hosted.is_hosted then
+	require "java"
+	local BackupIndexWriter = require "java:goodjava.lucene.backup.BackupIndexWriter"
+	BackupIndexWriter.backupDomains = {"localhost"}
+end
+
+--local postgres_spec = Hosted.postgres_spec()
 --logger.info(stringify(postgres_spec))
 
-local dir = Io.uri("site:/private/local/lucene")
+local dir = uri("site:/private/local/lucene")
 local Db = Lucene.index( dir, {
 	default_type = Lucene.type.english
 	default_fields = {"subject","content"}
-	postgres_spec = postgres_spec
+	--postgres_spec = postgres_spec
+	log_dir = uri("site:/private/local/lucene_log")
 } )
 
 --	this is how you index a field
 --	db.indexed_fields.post_date = Lucene.type.long
 
-Db.restore_from_postgres()
+--Db.restore_from_postgres()
+Db.restore_from_log()
 Db.update{
 	[1] = function()
 		logger.info "update"
diff -r 8066b8882732 -r 8dd8c556c449 src/goodjava/lucene/backup/Backup.java
--- a/src/goodjava/lucene/backup/Backup.java	Thu May 12 10:56:45 2022 -0600
+++ b/src/goodjava/lucene/backup/Backup.java	Thu May 26 21:10:02 2022 -0600
@@ -21,7 +21,7 @@
 import goodjava.lucene.logging.LogOutputStream;
 
 
-class Backup {
+final class Backup {
 	private static final Logger logger = LoggerFactory.getLogger(Backup.class);
 
 	private final File dir;
diff -r 8066b8882732 -r 8dd8c556c449 src/goodjava/lucene/backup/BackupIndexWriter.java
--- a/src/goodjava/lucene/backup/BackupIndexWriter.java	Thu May 12 10:56:45 2022 -0600
+++ b/src/goodjava/lucene/backup/BackupIndexWriter.java	Thu May 26 21:10:02 2022 -0600
@@ -4,6 +4,7 @@
 import java.io.InputStream;
 import java.io.IOException;
 import java.net.Socket;
+import java.net.ConnectException;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Map;
@@ -11,6 +12,7 @@
 import java.util.Arrays;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ExecutionException;
 import org.apache.lucene.search.SortField;
 import goodjava.io.IoUtils;
 import goodjava.rpc.RpcClient;
@@ -18,13 +20,13 @@
 import goodjava.rpc.RpcResult;
 import goodjava.rpc.RpcException;
 import goodjava.lucene.api.LuceneIndexWriter;
-import goodjava.logging.Logger;
-import goodjava.logging.LoggerFactory;
 import goodjava.lucene.logging.LoggingIndexWriter;
 import goodjava.lucene.logging.LogFile;
+import goodjava.logging.Logger;
+import goodjava.logging.LoggerFactory;
 
 
-public class BackupIndexWriter extends LoggingIndexWriter {
+public final class BackupIndexWriter extends LoggingIndexWriter {
 	private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class);
 	public static String[] backupDomains;
 	private final String name;
@@ -46,12 +48,12 @@
 		IoUtils.mkdirs(dir);
 	}
 
-	public synchronized void close() throws IOException {
+	@Override public synchronized void close() throws IOException {
 		super.close();
 		exec.shutdown();
 	}
 
-	public synchronized void commit() throws IOException {
+	@Override public synchronized void commit() throws IOException {
 		super.commit();
 		//sync();
 		if( !isSyncPending ) {
@@ -60,7 +62,7 @@
 		}
 	}
 
-	protected boolean doCheck(SortField sortField) throws IOException {
+	@Override protected boolean doCheck(SortField sortField) throws IOException {
 		boolean ok = super.doCheck(sortField);
 		if( ok )
 			runSyncWithChecksum();
@@ -68,17 +70,22 @@
 	}
 
 	public void runSync() {
-		try {
-			exec.submit(sync).get();
-		} catch(Exception e) {
-			throw new RuntimeException(e);
-		}
+		exec(sync);
 	}
 
 	public void runSyncWithChecksum() {
+		exec(syncWithChecksum);
+	}
+
+	private void exec(Runnable r) {
 		try {
-			exec.submit(syncWithChecksum).get();
-		} catch(Exception e) {
+			exec.submit(r).get();
+		} catch(InterruptedException e) {
+			throw new RuntimeException(e);
+		} catch(ExecutionException e) {
+			Throwable cause = e.getCause();
+			if( cause instanceof RuntimeException )
+				throw (RuntimeException)cause;
 			throw new RuntimeException(e);
 		}
 	}
@@ -87,6 +94,8 @@
 		public void run() {
 			try {
 				sync(false);
+			} catch(ConnectException e) {
+				logger.error("sync failed: "+e.getMessage());
 			} catch(IOException e) {
 				throw new RuntimeException(e);
 			}
@@ -97,6 +106,8 @@
 		public void run() {
 			try {
 				sync(true);
+			} catch(ConnectException e) {
+				logger.error("syncWithChecksum failed: "+e.getMessage());
 			} catch(IOException e) {
 				throw new RuntimeException(e);
 			}
@@ -159,7 +170,7 @@
 						throw new RuntimeException("status "+status);
 				}
 			} catch(RpcException e) {
-				logger.warn("",e);
+				logger.error("",e);
 			}
 			rpc.close();
 		}
diff -r 8066b8882732 -r 8dd8c556c449 src/goodjava/lucene/backup/BackupServer.java
--- a/src/goodjava/lucene/backup/BackupServer.java	Thu May 12 10:56:45 2022 -0600
+++ b/src/goodjava/lucene/backup/BackupServer.java	Thu May 26 21:10:02 2022 -0600
@@ -28,7 +28,7 @@
 import goodjava.logging.LoggerFactory;
 
 
-public class BackupServer {
+public final class BackupServer {
 	private static final Logger logger = LoggerFactory.getLogger(BackupServer.class);
 
 	public static int port = 9101;
diff -r 8066b8882732 -r 8dd8c556c449 src/luan/LuanJavaFunction.java
--- a/src/luan/LuanJavaFunction.java	Thu May 12 10:56:45 2022 -0600
+++ b/src/luan/LuanJavaFunction.java	Thu May 26 21:10:02 2022 -0600
@@ -235,7 +235,7 @@
 		return RTN_SAME;
 	}
 
-	private interface ArgConverter {
+	public interface ArgConverter {
 		public Object convert(Object obj) throws LuanException;
 	}
 
@@ -456,7 +456,7 @@
 		return a;
 	}
 
-	private static ArgConverter getArgConverter(Class cls) {
+	public static ArgConverter getArgConverter(Class cls) {
 		if( cls == Double.TYPE || cls.equals(Double.class) )
 			return ARG_DOUBLE;
 		if( cls == Float.TYPE || cls.equals(Float.class) )
diff -r 8066b8882732 -r 8dd8c556c449 src/luan/modules/JavaLuan.java
--- a/src/luan/modules/JavaLuan.java	Thu May 12 10:56:45 2022 -0600
+++ b/src/luan/modules/JavaLuan.java	Thu May 26 21:10:02 2022 -0600
@@ -166,9 +166,10 @@
 		throw new LuanException( "invalid index for java "+cls );
 	}
 
-	private static void setMember(Object obj,List<Member> members,Object value) {
+	private static void setMember(Object obj,List<Member> members,Object value) throws LuanException {
 		Field field = (Field)members.get(0);
 		try {
+/*
 			try {
 				field.set(obj,value);
 			} catch(IllegalArgumentException e) {
@@ -183,10 +184,16 @@
 						}
 					}
 				}
-				throw e;
+				throw new LuanException(e);
 			}
+*/
+			Class cls = field.getType();
+			LuanJavaFunction.ArgConverter ac = LuanJavaFunction.getArgConverter(cls);
+			field.set( obj, ac.convert(value) );
+		} catch(IllegalArgumentException e) {
+			throw new LuanException(e);
 		} catch(IllegalAccessException e) {
-			throw new RuntimeException(e);
+			throw new LuanException(e);
 		}
 	}
 
diff -r 8066b8882732 -r 8dd8c556c449 src/luan/modules/lucene/Lucene.luan
--- a/src/luan/modules/lucene/Lucene.luan	Thu May 12 10:56:45 2022 -0600
+++ b/src/luan/modules/lucene/Lucene.luan	Thu May 26 21:10:02 2022 -0600
@@ -71,7 +71,6 @@
 function Lucene.index(index_dir,options)
 	local index = {}
 	local options_name = options.name
-	options.name = nil
 	index.dir = index_dir
 	index_dir = get_file(index_dir)
 	options = options or {}
diff -r 8066b8882732 -r 8dd8c556c449 src/luan/modules/lucene/LuceneIndex.java
--- a/src/luan/modules/lucene/LuceneIndex.java	Thu May 12 10:56:45 2022 -0600
+++ b/src/luan/modules/lucene/LuceneIndex.java	Thu May 26 21:10:02 2022 -0600
@@ -81,6 +81,7 @@
 import goodjava.lucene.api.LuceneUtils;
 import goodjava.lucene.logging.LoggingIndexWriter;
 import goodjava.lucene.logging.OpDoer;
+import goodjava.lucene.backup.BackupIndexWriter;
 import goodjava.parser.ParseException;
 import luan.modules.Utils;
 import luan.Luan;
@@ -146,6 +147,9 @@
 	private boolean wasCreated;
 	private final File logDir;
 	private final long logTime;
+	private final String name;
+	private final String domain;
+	private final String password;
 
 	private LuceneIndex(Luan luan,File indexDir,LuanTable options)
 		throws LuanException, IOException, ClassNotFoundException, SQLException
@@ -159,8 +163,24 @@
 		LuanFunction supplementer = Utils.removeFunction(options,"supplementer");
 		logDir = (File)options.remove("log_dir");
 		logTime = (Long)options.remove("log_time");
+		name = (String)options.remove("name");
 		Utils.checkEmpty(options);
 
+		{
+			LuanTable module = (LuanTable)luan.require("luan:http/Http.luan");
+			String domain = (String)module.get(luan,"domain");
+			if( domain == null )
+				domain = "localhost";
+			this.domain = domain;
+		}
+		{
+			LuanTable module = (LuanTable)luan.require("luan:Io.luan");
+			String password = (String)module.get(luan,"password");
+			if( password == null )
+				password = "password";
+			this.password = password;
+		}
+
 		mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields);
 		mfp.fields.put( "type", STRING_FIELD_PARSER );
 		mfp.fields.put( "id", NumberFieldParser.LONG );
@@ -193,8 +213,17 @@
 		fsDir = FSDirectory.open(indexDir);
 		boolean wasCreated = !fsDir.getDirectory().exists();
 		writer = new LuceneIndexWriter(fsDir,config);
-		if( logDir != null )
-			writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime);
+		if( logDir != null ) {
+			if( BackupIndexWriter.backupDomains == null ) {
+				writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime);
+			} else {
+				String name = this.domain;
+				if( this.name != null )
+					name += "~" + this.name;
+				writer = new BackupIndexWriter((LuceneIndexWriter)writer,logDir,logTime,name,password);
+//qqq
+			}
+		}
 		reader = DirectoryReader.open(fsDir);
 		searcher = new IndexSearcher(reader);
 		initId();