Mercurial Hosting > luan
changeset 754:1a101ac9ea46
add lucene restore
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 17 Jul 2016 19:21:52 -0600 |
parents | 5e3970ccd86a |
children | ff865e954934 |
files | core/src/luan/modules/RpcLuan.java core/src/luan/modules/host/restore.luan lucene/src/luan/modules/lucene/Lucene.luan lucene/src/luan/modules/lucene/LuceneIndex.java |
diffstat | 4 files changed, 75 insertions(+), 47 deletions(-) [+] |
line wrap: on
line diff
--- a/core/src/luan/modules/RpcLuan.java Fri Jul 15 17:35:50 2016 -0600 +++ b/core/src/luan/modules/RpcLuan.java Sun Jul 17 19:21:52 2016 -0600 @@ -5,6 +5,7 @@ import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.InputStreamReader; +import java.io.FilterInputStream; import java.io.IOException; import java.io.EOFException; import java.net.Socket; @@ -76,12 +77,13 @@ } catch(LuanException e) { writeBoolean(out,false); writeString(out,e.getFullMessage()); + out.flush(); return; } writeBoolean(out,true); writeObjs(out,luan,rtn); + out.flush(); } finally { - out.flush(); socket.close(); } } @@ -267,12 +269,12 @@ if(this.close) close.b = false; } - private void close() throws IOException { - if(close) in.close(); - } - @Override public InputStream inputStream() { - return in; + return new FilterInputStream(in) { + @Override public void close() throws IOException { + if(close) super.close(); + } + }; } @Override public String to_string() { @@ -283,18 +285,6 @@ throw new UnsupportedOperationException(); } - @Override public String read_text() throws IOException { - String rtn = Utils.readAll(new InputStreamReader(in)); - close(); - return rtn; - } - - @Override public byte[] read_binary() throws IOException { - byte[] rtn = Utils.readAll(in); - close(); - return rtn; - } - @Override public boolean exists() { return true; }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/core/src/luan/modules/host/restore.luan Sun Jul 17 19:21:52 2016 -0600 @@ -0,0 +1,19 @@ +local Luan = require "luan:Luan.luan" +local error = Luan.error +local Io = require "luan:Io.luan" +local print = Io.print or error() +local uri = Io.uri or error() +local Hosting = require "luan:host/Hosting.luan" + +if #{...} ~= 2 then + Io.stderr.write "usage: luan luan:host/restore.luan domain password\n" + return +end + +local domain, password = ... + +local zip_file = uri("file:backup.zip") +zip_file.exists() or error "backup.zip not found" +Hosting.caller(domain).lucene_restore(password,zip_file) + +print("restored lucene from backup.zip to "..domain)
--- a/lucene/src/luan/modules/lucene/Lucene.luan Fri Jul 15 17:35:50 2016 -0600 +++ b/lucene/src/luan/modules/lucene/Lucene.luan Sun Jul 17 19:21:52 2016 -0600 @@ -9,6 +9,8 @@ local String = require "luan:String.luan" local matches = String.matches or error() local Rpc = require "luan:Rpc.luan" +local Thread = require "luan:Thread.luan" +local synchronized = Thread.synchronized or error() local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex" local NumberFieldParser = require "java:luan.modules.lucene.queryparser.NumberFieldParser" local StringFieldParser = require "java:luan.modules.lucene.queryparser.StringFieldParser" @@ -113,7 +115,23 @@ end ) end + function index.restore(zip_file) + local lucene_dir = uri("file:"..index.dir) + local before_restore = lucene_dir.parent().child("before_restore.zip") + index.zip(before_restore) + java_index.close() + lucene_dir.delete() + zip_file.unzip(lucene_dir.parent().to_string()) + java_index.reopen() + end + index.restore = synchronized(index.restore) + + local function multi_error() + error "multiple lucene instances" + end + if Rpc.functions.backup == nil then + function Rpc.functions.lucene_backup(password) if Io.password ~= password then error "wrong password" @@ -122,10 +140,17 @@ index.zip(zip_file) return zip_file end + + function Rpc.functions.lucene_restore(password,zip_file) + if Io.password ~= password then + error "wrong password" + end + index.restore(zip_file) + end + else - function Rpc.functions.lucene_backup() - error "multiple lucene instance" - end + Rpc.functions.lucene_backup = multi_error + Rpc.functions.lucene_restore = multi_error end return index
--- a/lucene/src/luan/modules/lucene/LuceneIndex.java Fri Jul 15 17:35:50 2016 -0600 +++ b/lucene/src/luan/modules/lucene/LuceneIndex.java Sun Jul 17 19:21:52 2016 -0600 @@ -83,20 +83,21 @@ private static final String FLD_NEXT_ID = "nextId"; public static final StringFieldParser STRING_FIELD_PARSER = new StringFieldParser(new KeywordAnalyzer()); + private static final Version version = Version.LUCENE_4_9; private final ReentrantLock writeLock = new ReentrantLock(); private final File indexDir; - final SnapshotDeletionPolicy snapshotDeletionPolicy; - private final IndexWriter writer; + private SnapshotDeletionPolicy snapshotDeletionPolicy; + private IndexWriter writer; private DirectoryReader reader; private IndexSearcher searcher; private final ThreadLocal<IndexSearcher> threadLocalSearcher = new ThreadLocal<IndexSearcher>(); - private boolean isClosed = false; + private boolean isClosed = true; private final MultiFieldParser mfp; public final LuanTable indexed_only_fields = new LuanTable(); private final Analyzer analyzer; private static ConcurrentMap<File,AtomicInteger> globalWriteCounters = new ConcurrentHashMap<File,AtomicInteger>(); - private final File fileDir; + private File fileDir; private int writeCount; public LuceneIndex(LuanState luan,String indexDirStr,FieldParser defaultFieldParser,String[] defaultFields) throws LuanException, IOException { @@ -105,23 +106,28 @@ mfp.fields.put( "id", NumberFieldParser.LONG ); File indexDir = new File(indexDirStr); this.indexDir = indexDir; - FSDirectory dir = FSDirectory.open(indexDir); - fileDir = dir.getDirectory(); - globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger()); - Version version = Version.LUCENE_4_9; Analyzer analyzer = STRING_FIELD_PARSER.analyzer; if( defaultFieldParser instanceof StringFieldParser ) { StringFieldParser sfp = (StringFieldParser)defaultFieldParser; analyzer = sfp.analyzer; } this.analyzer = analyzer; + luan.onClose(this); + reopen(); + } + + public void reopen() throws LuanException, IOException { + if( !isClosed ) throw new RuntimeException(); + isClosed = false; IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); conf.setIndexDeletionPolicy(snapshotDeletionPolicy); + FSDirectory dir = FSDirectory.open(indexDir); + fileDir = dir.getDirectory(); + globalWriteCounters.putIfAbsent(fileDir,new AtomicInteger()); writer = new IndexWriter(dir,conf); writer.commit(); // commit index creation reader = DirectoryReader.open(dir); - luan.onClose(this); searcher = new IndexSearcher(reader); initId(); } @@ -232,29 +238,17 @@ } - private long id = 0; - private long idLim = 0; + private long id; + private long idLim; private final int idBatch = 10; private void initId() throws LuanException, IOException { TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); -/* - // tmp hack - if( td.totalHits == 0 ) { - td = searcher.search(new TermQuery(new Term("type index","next_id")),1); - if( td.totalHits == 1 ) { - long idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue(); - LuanTable doc = new LuanTable(); - doc.rawPut( "type", "next_id" ); - doc.rawPut( FLD_NEXT_ID, idLim ); - writer.addDocument(toLucene(luan,doc)); - writer.commit(); - } - } -*/ switch(td.totalHits) { case 0: - break; // do nothing + id = 0; + idLim = 0; + break; case 1: idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue(); id = idLim;