Mercurial Hosting > luan
changeset 1672:8dd8c556c449
backup work
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Thu, 26 May 2022 21:10:02 -0600 |
parents | 8066b8882732 |
children | 1b9f9fdb3b41 |
files | .hgignore backup/server.luan examples/blog/src/lib/Db.luan src/goodjava/lucene/backup/Backup.java src/goodjava/lucene/backup/BackupIndexWriter.java src/goodjava/lucene/backup/BackupServer.java src/luan/LuanJavaFunction.java src/luan/modules/JavaLuan.java src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java |
diffstat | 10 files changed, 133 insertions(+), 29 deletions(-) [+] |
line wrap: on
line diff
--- a/.hgignore Thu May 12 10:56:45 2022 -0600 +++ b/.hgignore Thu May 26 21:10:02 2022 -0600 @@ -14,3 +14,5 @@ host/logs/ host/started.lock push.sh +backup/logs/ +backup/backups/
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/backup/server.luan Thu May 26 21:10:02 2022 -0600 @@ -0,0 +1,47 @@ +local Luan = require "luan:Luan.luan" +local error = Luan.error +local ipairs = Luan.ipairs or error() +local Io = require "luan:Io.luan" + +require "java" +local File = require "java:java.io.File" +local BackupServer = require "java:goodjava.lucene.backup.BackupServer" +local Logging = require "luan:logging/Logging.luan" + +local log_to_console = false +for _, arg in ipairs{...} do + if arg == "console" then + log_to_console = true + end +end +if not log_to_console then + local FileWriter = require "java:java.io.FileWriter" + local LuanLogger = require "java:luan.modules.logging.LuanLogger" + local Layouts = require "java:goodjava.logger.Layouts" + local DateLayout = require "java:goodjava.logger.DateLayout" + local ListLayout = require "java:goodjava.logger.ListLayout" + local Level = require "java:goodjava.logger.Level" + local WriterAppender = require "java:goodjava.logger.WriterAppender" + local LevelAppender = require "java:goodjava.logger.LevelAppender" + local ListAppender = require "java:goodjava.logger.ListAppender" + + local layout = ListLayout.new(DateLayout.new("yyyy-MM-dd HH:mm:ss,SSS")," ",Layouts.LEVEL_PADDED," ",Layouts.LOGGER," - ",Layouts.MESSAGE,"\n",Layouts.THROWABLE) + + local function new_appender(file,level) + local writer = FileWriter.new(file,true) + local appender = WriterAppender.new(layout,writer) + appender = LevelAppender.new(appender,level) + return appender + end + + Io.uri("file:logs").mkdir() + local err = new_appender("logs/error.log",Level.ERROR) + local warn = new_appender("logs/warn.log",Level.WARN) + local info = new_appender("logs/info.log",Level.INFO) + local appender = ListAppender.new(err,warn,info) + LuanLogger.configure(appender) +end + +local dir = File.new("backups") +local server = BackupServer.new(dir) +server.start()
--- a/examples/blog/src/lib/Db.luan Thu May 12 10:56:45 2022 -0600 +++ b/examples/blog/src/lib/Db.luan Thu May 26 21:10:02 2022 -0600 @@ -3,27 +3,36 @@ local stringify = Luan.stringify or error() local Lucene = require "luan:lucene/Lucene.luan" local Io = require "luan:Io.luan" -local Hosted = require "luan:host/Hosted.luan" +local uri = Io.uri or error() local Time = require "luan:Time.luan" local Thread = require "luan:Thread.luan" +local Hosted = require "luan:host/Hosted.luan" local Logging = require "luan:logging/Logging.luan" local logger = Logging.logger "Db" -local postgres_spec = Hosted.postgres_spec() +if not Hosted.is_hosted then + require "java" + local BackupIndexWriter = require "java:goodjava.lucene.backup.BackupIndexWriter" + BackupIndexWriter.backupDomains = {"localhost"} +end + +--local postgres_spec = Hosted.postgres_spec() --logger.info(stringify(postgres_spec)) -local dir = Io.uri("site:/private/local/lucene") +local dir = uri("site:/private/local/lucene") local Db = Lucene.index( dir, { default_type = Lucene.type.english default_fields = {"subject","content"} - postgres_spec = postgres_spec + --postgres_spec = postgres_spec + log_dir = uri("site:/private/local/lucene_log") } ) -- this is how you index a field -- db.indexed_fields.post_date = Lucene.type.long -Db.restore_from_postgres() +--Db.restore_from_postgres() +Db.restore_from_log() Db.update{ [1] = function() logger.info "update"
--- a/src/goodjava/lucene/backup/Backup.java Thu May 12 10:56:45 2022 -0600 +++ b/src/goodjava/lucene/backup/Backup.java Thu May 26 21:10:02 2022 -0600 @@ -21,7 +21,7 @@ import goodjava.lucene.logging.LogOutputStream; -class Backup { +final class Backup { private static final Logger logger = LoggerFactory.getLogger(Backup.class); private final File dir;
--- a/src/goodjava/lucene/backup/BackupIndexWriter.java Thu May 12 10:56:45 2022 -0600 +++ b/src/goodjava/lucene/backup/BackupIndexWriter.java Thu May 26 21:10:02 2022 -0600 @@ -4,6 +4,7 @@ import java.io.InputStream; import java.io.IOException; import java.net.Socket; +import java.net.ConnectException; import java.util.List; import java.util.ArrayList; import java.util.Map; @@ -11,6 +12,7 @@ import java.util.Arrays; import java.util.concurrent.Executors; import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; import org.apache.lucene.search.SortField; import goodjava.io.IoUtils; import goodjava.rpc.RpcClient; @@ -18,13 +20,13 @@ import goodjava.rpc.RpcResult; import goodjava.rpc.RpcException; import goodjava.lucene.api.LuceneIndexWriter; -import goodjava.logging.Logger; -import goodjava.logging.LoggerFactory; import goodjava.lucene.logging.LoggingIndexWriter; import goodjava.lucene.logging.LogFile; +import goodjava.logging.Logger; +import goodjava.logging.LoggerFactory; -public class BackupIndexWriter extends LoggingIndexWriter { +public final class BackupIndexWriter extends LoggingIndexWriter { private static final Logger logger = LoggerFactory.getLogger(BackupIndexWriter.class); public static String[] backupDomains; private final String name; @@ -46,12 +48,12 @@ IoUtils.mkdirs(dir); } - public synchronized void close() throws IOException { + @Override public synchronized void close() throws IOException { super.close(); exec.shutdown(); } - public synchronized void commit() throws IOException { + @Override public synchronized void commit() throws IOException { super.commit(); //sync(); if( !isSyncPending ) { @@ -60,7 +62,7 @@ } } - protected boolean doCheck(SortField sortField) throws IOException { + @Override protected boolean doCheck(SortField sortField) throws IOException { boolean ok = super.doCheck(sortField); if( ok ) runSyncWithChecksum(); @@ -68,17 +70,22 @@ } public void runSync() { - try { - exec.submit(sync).get(); - } catch(Exception e) { - throw new RuntimeException(e); - } + exec(sync); } public void runSyncWithChecksum() { + exec(syncWithChecksum); + } + + private void exec(Runnable r) { try { - exec.submit(syncWithChecksum).get(); - } catch(Exception e) { + exec.submit(r).get(); + } catch(InterruptedException e) { + throw new RuntimeException(e); + } catch(ExecutionException e) { + Throwable cause = e.getCause(); + if( cause instanceof RuntimeException ) + throw (RuntimeException)cause; throw new RuntimeException(e); } } @@ -87,6 +94,8 @@ public void run() { try { sync(false); + } catch(ConnectException e) { + logger.error("sync failed: "+e.getMessage()); } catch(IOException e) { throw new RuntimeException(e); } @@ -97,6 +106,8 @@ public void run() { try { sync(true); + } catch(ConnectException e) { + logger.error("syncWithChecksum failed: "+e.getMessage()); } catch(IOException e) { throw new RuntimeException(e); } @@ -159,7 +170,7 @@ throw new RuntimeException("status "+status); } } catch(RpcException e) { - logger.warn("",e); + logger.error("",e); } rpc.close(); }
--- a/src/goodjava/lucene/backup/BackupServer.java Thu May 12 10:56:45 2022 -0600 +++ b/src/goodjava/lucene/backup/BackupServer.java Thu May 26 21:10:02 2022 -0600 @@ -28,7 +28,7 @@ import goodjava.logging.LoggerFactory; -public class BackupServer { +public final class BackupServer { private static final Logger logger = LoggerFactory.getLogger(BackupServer.class); public static int port = 9101;
--- a/src/luan/LuanJavaFunction.java Thu May 12 10:56:45 2022 -0600 +++ b/src/luan/LuanJavaFunction.java Thu May 26 21:10:02 2022 -0600 @@ -235,7 +235,7 @@ return RTN_SAME; } - private interface ArgConverter { + public interface ArgConverter { public Object convert(Object obj) throws LuanException; } @@ -456,7 +456,7 @@ return a; } - private static ArgConverter getArgConverter(Class cls) { + public static ArgConverter getArgConverter(Class cls) { if( cls == Double.TYPE || cls.equals(Double.class) ) return ARG_DOUBLE; if( cls == Float.TYPE || cls.equals(Float.class) )
--- a/src/luan/modules/JavaLuan.java Thu May 12 10:56:45 2022 -0600 +++ b/src/luan/modules/JavaLuan.java Thu May 26 21:10:02 2022 -0600 @@ -166,9 +166,10 @@ throw new LuanException( "invalid index for java "+cls ); } - private static void setMember(Object obj,List<Member> members,Object value) { + private static void setMember(Object obj,List<Member> members,Object value) throws LuanException { Field field = (Field)members.get(0); try { +/* try { field.set(obj,value); } catch(IllegalArgumentException e) { @@ -183,10 +184,16 @@ } } } - throw e; + throw new LuanException(e); } +*/ + Class cls = field.getType(); + LuanJavaFunction.ArgConverter ac = LuanJavaFunction.getArgConverter(cls); + field.set( obj, ac.convert(value) ); + } catch(IllegalArgumentException e) { + throw new LuanException(e); } catch(IllegalAccessException e) { - throw new RuntimeException(e); + throw new LuanException(e); } }
--- a/src/luan/modules/lucene/Lucene.luan Thu May 12 10:56:45 2022 -0600 +++ b/src/luan/modules/lucene/Lucene.luan Thu May 26 21:10:02 2022 -0600 @@ -71,7 +71,6 @@ function Lucene.index(index_dir,options) local index = {} local options_name = options.name - options.name = nil index.dir = index_dir index_dir = get_file(index_dir) options = options or {}
--- a/src/luan/modules/lucene/LuceneIndex.java Thu May 12 10:56:45 2022 -0600 +++ b/src/luan/modules/lucene/LuceneIndex.java Thu May 26 21:10:02 2022 -0600 @@ -81,6 +81,7 @@ import goodjava.lucene.api.LuceneUtils; import goodjava.lucene.logging.LoggingIndexWriter; import goodjava.lucene.logging.OpDoer; +import goodjava.lucene.backup.BackupIndexWriter; import goodjava.parser.ParseException; import luan.modules.Utils; import luan.Luan; @@ -146,6 +147,9 @@ private boolean wasCreated; private final File logDir; private final long logTime; + private final String name; + private final String domain; + private final String password; private LuceneIndex(Luan luan,File indexDir,LuanTable options) throws LuanException, IOException, ClassNotFoundException, SQLException @@ -159,8 +163,24 @@ LuanFunction supplementer = Utils.removeFunction(options,"supplementer"); logDir = (File)options.remove("log_dir"); logTime = (Long)options.remove("log_time"); + name = (String)options.remove("name"); Utils.checkEmpty(options); + { + LuanTable module = (LuanTable)luan.require("luan:http/Http.luan"); + String domain = (String)module.get(luan,"domain"); + if( domain == null ) + domain = "localhost"; + this.domain = domain; + } + { + LuanTable module = (LuanTable)luan.require("luan:Io.luan"); + String password = (String)module.get(luan,"password"); + if( password == null ) + password = "password"; + this.password = password; + } + mfp = defaultFieldParser==null ? new MultiFieldParser() : new MultiFieldParser(defaultFieldParser,defaultFields); mfp.fields.put( "type", STRING_FIELD_PARSER ); mfp.fields.put( "id", NumberFieldParser.LONG ); @@ -193,8 +213,17 @@ fsDir = FSDirectory.open(indexDir); boolean wasCreated = !fsDir.getDirectory().exists(); writer = new LuceneIndexWriter(fsDir,config); - if( logDir != null ) - writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime); + if( logDir != null ) { + if( BackupIndexWriter.backupDomains == null ) { + writer = new LoggingIndexWriter((LuceneIndexWriter)writer,logDir,logTime); + } else { + String name = this.domain; + if( this.name != null ) + name += "~" + this.name; + writer = new BackupIndexWriter((LuceneIndexWriter)writer,logDir,logTime,name,password); +//qqq + } + } reader = DirectoryReader.open(fsDir); searcher = new IndexSearcher(reader); initId();