Mercurial Hosting > luan
changeset 1434:56fb5cd8228d
cache compiled code in temp files
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Sun, 29 Dec 2019 15:25:07 -0700 |
parents | 5f038be65271 |
children | 3aae37efbf40 |
files | src/luan/Luan.java src/luan/impl/Compiled.java src/luan/impl/LuanCompiler.java src/luan/impl/LuanJavaCompiler.java src/luan/impl/LuanParser.java src/luan/modules/BasicLuan.java src/luan/modules/Boot.luan src/luan/modules/Luan.luan src/luan/modules/PackageLuan.java src/luan/modules/http/LuanHandler.java src/luan/modules/lucene/PostgresBackup.java website/src/manual.html |
diffstat | 12 files changed, 269 insertions(+), 155 deletions(-) [+] |
line wrap: on
line diff
--- a/src/luan/Luan.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/Luan.java Sun Dec 29 15:25:07 2019 -0700 @@ -67,7 +67,7 @@ } public Object eval(String cmd,Object... args) throws LuanException { - return load(cmd,"eval").call(args); + return load(cmd,"eval",false).call(args); } public Object require(String modName) throws LuanException { @@ -182,7 +182,7 @@ { try { String src = IoLuan.classpath(luan,classpath).read_text(); - return luan.load(src,"classpath:"+classpath); + return luan.load(src,"classpath:"+classpath,true); } catch(IOException e) { throw new RuntimeException(e); } @@ -280,16 +280,16 @@ throw new LuanException("attempt to call a " + Luan.type(obj) + " value" ); } - public LuanFunction load(String text,String sourceName,LuanTable env) + public LuanFunction load(String text,String sourceName,boolean persist,LuanTable env) throws LuanException { - return LuanCompiler.compile(this,text,sourceName,env); + return LuanCompiler.compile(this,text,sourceName,persist,env); } - public LuanFunction load(String text,String sourceName) + public LuanFunction load(String text,String sourceName,boolean persist) throws LuanException { - return load(text,sourceName,null); + return load(text,sourceName,persist,null); } public static Object toJava(Object obj) throws LuanException {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/impl/Compiled.java Sun Dec 29 15:25:07 2019 -0700 @@ -0,0 +1,191 @@ +package luan.impl; + +import java.io.OutputStream; +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.URI; +import java.util.Collections; +import java.util.Map; +import java.util.HashMap; +import java.util.Set; +import java.util.HashSet; +import javax.tools.FileObject; +import javax.tools.JavaFileObject; +import javax.tools.SimpleJavaFileObject; +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; +import javax.tools.JavaFileManager; +import javax.tools.StandardJavaFileManager; +import javax.tools.ForwardingJavaFileManager; +import goodjava.logging.Logger; +import goodjava.logging.LoggerFactory; + + +final class Compiled { + private static final Logger logger = LoggerFactory.getLogger(Compiled.class); + + private static class MyJavaFileObject extends SimpleJavaFileObject { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + MyJavaFileObject() { + super(URI.create("whatever"),JavaFileObject.Kind.CLASS); + } + + @Override public OutputStream openOutputStream() { + return baos; + } + + byte[] byteCode(String sourceName) { + byte[] byteCode = baos.toByteArray(); + final int len = sourceName.length(); + int max = byteCode.length-len-3; + outer: + for( int i=0; true; i++ ) { + if( i > max ) + throw new RuntimeException("len="+len); + if( byteCode[i]==1 && (byteCode[i+1] << 8 | 0xFF & byteCode[i+2]) == len ) { + for( int j=i+3; j<i+3+len; j++ ) { + if( byteCode[j] != '$' ) + continue outer; + } + System.arraycopy(sourceName.getBytes(),0,byteCode,i+3,len); + break; + } + } + return byteCode; + } + } + + static Compiled compile(final String className,final String sourceName,final String code) { + final int len = sourceName.length(); + StringBuilder sb = new StringBuilder(sourceName); + for( int i=0; i<len; i++ ) + sb.setCharAt(i,'$'); + JavaFileObject sourceFile = new SimpleJavaFileObject(URI.create(sb.toString()),JavaFileObject.Kind.SOURCE) { + @Override public CharSequence getCharContent(boolean ignoreEncodingErrors) { + return code; + } + @Override public String getName() { + return sourceName; + } + @Override public boolean isNameCompatible(String simpleName,JavaFileObject.Kind kind) { + return true; + } + }; + final Map<String,MyJavaFileObject> map = new HashMap<String,MyJavaFileObject>(); + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + StandardJavaFileManager sjfm = compiler.getStandardFileManager(null,null,null); + ForwardingJavaFileManager fjfm = new ForwardingJavaFileManager(sjfm) { + @Override public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location, String className, JavaFileObject.Kind kind, FileObject sibling) throws IOException { + if( map.containsKey(className) ) + throw new RuntimeException(className); + MyJavaFileObject classFile = new MyJavaFileObject(); + map.put(className,classFile); + return classFile; + } + }; + StringWriter out = new StringWriter(); + boolean b = compiler.getTask(out, fjfm, null, null, null, Collections.singletonList(sourceFile)).call(); + if( !b ) + throw new RuntimeException("\n"+out+"\ncode:\n"+code+"\n"); + Map<String,byte[]> map2 = new HashMap<String,byte[]>(); + for( Map.Entry<String,MyJavaFileObject> entry : map.entrySet() ) { + map2.put( entry.getKey(), entry.getValue().byteCode(sourceName) ); + } + return new Compiled(className,map2); + } + + + private final String className; + private final Map<String,byte[]> map; + private final Set<String> set = new HashSet<String>(); + + private Compiled(String className,Map<String,byte[]> map) { + this.className = className; + this.map = map; + } + + Class loadClass() { + try { + ClassLoader cl = new ClassLoader() { + @Override protected Class<?> findClass(String name) throws ClassNotFoundException { + if( !set.add(name) ) + logger.error("dup "+name); + byte[] byteCode = map.get(name); + if( byteCode != null ) { + return defineClass(name, byteCode, 0, byteCode.length); + } + return super.findClass(name); + } + }; + return cl.loadClass(className); + } catch(ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + + private static final int VERSION = 1; + private static final File tmpDir; + static { + File f = new File(System.getProperty("java.io.tmpdir")); + tmpDir = new File(f,"luan"); + tmpDir.mkdir(); + if( !tmpDir.exists() ) + throw new RuntimeException(); + } + + static Compiled load(String fileName,String key) { + try { + File f = new File(tmpDir,fileName); + if( !f.exists() ) + return null; + DataInputStream in = new DataInputStream(new FileInputStream(f)); + if( in.readInt() != VERSION ) + return null; + if( !in.readUTF().equals(key) ) + return null; + String className = in.readUTF(); + int n = in.readInt(); + Map<String,byte[]> map = new HashMap<String,byte[]>(); + for( int i=0; i<n; i++ ) { + String s = in.readUTF(); + int len = in.readInt(); + byte[] a = new byte[len]; + in.readFully(a); + map.put(s,a); + } + in.close(); + return new Compiled(className,map); + } catch(IOException e) { + logger.error("load failed",e); + return null; + } + } + + void save(String fileName,String key) { + try { + File f = new File(tmpDir,fileName); + DataOutputStream out = new DataOutputStream(new FileOutputStream(f)); + out.writeInt(VERSION); + out.writeUTF(key); + out.writeUTF(className); + out.writeInt(map.size()); + for( Map.Entry<String,byte[]> entry : map.entrySet() ) { + out.writeUTF( entry.getKey() ); + byte[] a = entry.getValue(); + out.writeInt(a.length); + out.write(a,0,a.length); + } + out.close(); + } catch(IOException e) { + logger.error("save failed",e); + } + } +}
--- a/src/luan/impl/LuanCompiler.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/impl/LuanCompiler.java Sun Dec 29 15:25:07 2019 -0700 @@ -2,8 +2,11 @@ import java.lang.ref.WeakReference; import java.lang.reflect.InvocationTargetException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.Map; import java.util.HashMap; +import java.util.Base64; import luan.Luan; import luan.LuanFunction; import luan.LuanException; @@ -16,8 +19,12 @@ public final class LuanCompiler { private static final Map<String,WeakReference<Class>> map = new HashMap<String,WeakReference<Class>>(); - public static LuanFunction compile(Luan luan,String sourceText,String sourceName,LuanTable env) throws LuanException { - Class fnClass = env==null ? getClass(sourceText,sourceName) : getClass(sourceText,sourceName,env); + public static LuanFunction compile(Luan luan,String sourceText,String sourceName,boolean persist,LuanTable env) + throws LuanException + { + if( persist && env!=null ) + throw new LuanException("can't persist with env"); + Class fnClass = persist ? getClass(sourceText,sourceName) : getClass(sourceText,sourceName,env); boolean javaOk = false; if( env != null && env.closure != null ) javaOk = env.closure.javaOk; @@ -41,20 +48,45 @@ return closure; } - private static synchronized Class getClass(String sourceText,String sourceName) throws LuanException { + private static synchronized Class getClass(String sourceText,String sourceName) + throws LuanException + { String key = sourceName + "~~~" + sourceText; WeakReference<Class> ref = map.get(key); if( ref != null ) { Class cls = ref.get(); if( cls != null ) return cls; + } + String fileName; + try { + byte[] a = MessageDigest.getInstance("MD5").digest(key.getBytes()); + String s = Base64.getUrlEncoder().encodeToString(a); +//System.err.println("qqqqqqqqqq "+s); + fileName = s + ".luanc"; + } catch(NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + Compiled compiled = Compiled.load(fileName,key); + //Compiled compiled = null; + if( compiled==null ) { + compiled = getCompiled(sourceText,sourceName,null); + compiled.save(fileName,key); } - Class cls = getClass(sourceText,sourceName,null); + Class cls = compiled.loadClass(); map.put(key,new WeakReference<Class>(cls)); return cls; } - private static Class getClass(String sourceText,String sourceName,LuanTable env) throws LuanException { + private static Class getClass(String sourceText,String sourceName,LuanTable env) + throws LuanException + { + return getCompiled(sourceText,sourceName,env).loadClass(); + } + + private static Compiled getCompiled(String sourceText,String sourceName,LuanTable env) + throws LuanException + { LuanParser parser = new LuanParser(sourceText,sourceName); parser.addVar( "require" ); if( env != null ) parser.addVar( "_ENV" ); @@ -66,7 +98,9 @@ } } - public static String toJava(String sourceText,String sourceName) throws LuanException { + public static String toJava(String sourceText,String sourceName) + throws LuanException + { LuanParser parser = new LuanParser(sourceText,sourceName); parser.addVar( "require" ); try {
--- a/src/luan/impl/LuanJavaCompiler.java Tue Dec 24 17:57:47 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,100 +0,0 @@ -package luan.impl; - -import java.io.OutputStream; -import java.io.ByteArrayOutputStream; -import java.io.StringWriter; -import java.io.IOException; -import java.net.URI; -import java.util.Collections; -import java.util.Map; -import java.util.HashMap; -import javax.tools.FileObject; -import javax.tools.JavaFileObject; -import javax.tools.SimpleJavaFileObject; -import javax.tools.JavaCompiler; -import javax.tools.ToolProvider; -import javax.tools.JavaFileManager; -import javax.tools.StandardJavaFileManager; -import javax.tools.ForwardingJavaFileManager; - - -public final class LuanJavaCompiler { - private LuanJavaCompiler() {} // never - - private static class MyJavaFileObject extends SimpleJavaFileObject { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - MyJavaFileObject() { - super(URI.create("whatever"),JavaFileObject.Kind.CLASS); - } - - @Override public OutputStream openOutputStream() { - return baos; - } - - byte[] byteCode(String sourceName) { - byte[] byteCode = baos.toByteArray(); - final int len = sourceName.length(); - int max = byteCode.length-len-3; - outer: - for( int i=0; true; i++ ) { - if( i > max ) - throw new RuntimeException("len="+len); - if( byteCode[i]==1 && (byteCode[i+1] << 8 | 0xFF & byteCode[i+2]) == len ) { - for( int j=i+3; j<i+3+len; j++ ) { - if( byteCode[j] != '$' ) - continue outer; - } - System.arraycopy(sourceName.getBytes(),0,byteCode,i+3,len); - break; - } - } - return byteCode; - } - } - - public static Class compile(final String className,final String sourceName,final String code) throws ClassNotFoundException { - final int len = sourceName.length(); - StringBuilder sb = new StringBuilder(sourceName); - for( int i=0; i<len; i++ ) - sb.setCharAt(i,'$'); - JavaFileObject sourceFile = new SimpleJavaFileObject(URI.create(sb.toString()),JavaFileObject.Kind.SOURCE) { - @Override public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return code; - } - @Override public String getName() { - return sourceName; - } - @Override public boolean isNameCompatible(String simpleName,JavaFileObject.Kind kind) { - return true; - } - }; - final Map<String,MyJavaFileObject> map = new HashMap<String,MyJavaFileObject>(); - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - StandardJavaFileManager sjfm = compiler.getStandardFileManager(null,null,null); - ForwardingJavaFileManager fjfm = new ForwardingJavaFileManager(sjfm) { - @Override public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location, String className, JavaFileObject.Kind kind, FileObject sibling) throws IOException { - if( map.containsKey(className) ) - throw new RuntimeException(className); - MyJavaFileObject classFile = new MyJavaFileObject(); - map.put(className,classFile); - return classFile; - } - }; - StringWriter out = new StringWriter(); - boolean b = compiler.getTask(out, fjfm, null, null, null, Collections.singletonList(sourceFile)).call(); - if( !b ) - throw new RuntimeException("\n"+out+"\ncode:\n"+code+"\n"); - ClassLoader cl = new ClassLoader() { - @Override protected Class<?> findClass(String name) throws ClassNotFoundException { - MyJavaFileObject jfo = map.get(name); - if( jfo != null ) { - byte[] byteCode = jfo.byteCode(sourceName); - return defineClass(name, byteCode, 0, byteCode.length); - } - return super.findClass(name); - } - }; - return cl.loadClass(className); - } -}
--- a/src/luan/impl/LuanParser.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/impl/LuanParser.java Sun Dec 29 15:25:07 2019 -0700 @@ -168,7 +168,6 @@ } - private static AtomicInteger classCounter = new AtomicInteger(); private int innerCounter = 0; private final List<Inner> inners = new ArrayList<Inner>(); @@ -237,20 +236,16 @@ // return toFnExp( stmt, frame.upValueSymbols, name ); } - Class RequiredModule() throws ParseException { + Compiled RequiredModule() throws ParseException { GetRequiredModule(); - String className = "EXP" + classCounter.incrementAndGet(); + String className = "EXP"; String classCode = toFnString( top, frame.upValueSymbols, className, inners ); - try { - return LuanJavaCompiler.compile("luan.impl."+className,parser.sourceName,classCode); - } catch(ClassNotFoundException e) { - throw new RuntimeException(e); - } + return Compiled.compile("luan.impl."+className,parser.sourceName,classCode); } String RequiredModuleSource() throws ParseException { GetRequiredModule(); - String className = "EXP" + classCounter.incrementAndGet(); + String className = "EXP"; return toFnString( top, frame.upValueSymbols, className, inners ); }
--- a/src/luan/modules/BasicLuan.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/modules/BasicLuan.java Sun Dec 29 15:25:07 2019 -0700 @@ -23,12 +23,12 @@ return Luan.type(obj); } - public static LuanFunction load(Luan luan,String text,String sourceName,LuanTable env) + public static LuanFunction load(Luan luan,String text,String sourceName,boolean persist,LuanTable env) throws LuanException { Utils.checkNotNull(text); Utils.checkNotNull(sourceName,1); - return luan.load(text,sourceName,env); + return luan.load(text,sourceName,persist,env); } /* public static LuanFunction load_file(Luan luan,String fileName) throws LuanException {
--- a/src/luan/modules/Boot.luan Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/modules/Boot.luan Sun Dec 29 15:25:07 2019 -0700 @@ -245,11 +245,11 @@ return nil end local src = u.read_text() - return load(src,file) + return load(src,file,true) elseif type(file) == "table" and file.read_text ~= nil then local src = file.read_text() local src_file = file.uri_string or file.to_uri_string() - return load(src,src_file) + return load(src,src_file,true) else error("bad argument, expected string or uri table but got "..type(file)) end
--- a/src/luan/modules/Luan.luan Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/modules/Luan.luan Sun Dec 29 15:25:07 2019 -0700 @@ -9,8 +9,9 @@ Luan.get_metatable = BasicLuan.get_metatable Luan.hash_code = BasicLuan.hash_code Luan.ipairs = BasicLuan.ipairs -Luan.load = BasicLuan.load -Luan.load_file = Boot.load_file +local java_load = BasicLuan.load +local load_file = Boot.load_file +Luan.load_file = load_file Luan.new_error = BasicLuan.new_error Luan.pairs = BasicLuan.pairs Luan.pcall = BasicLuan.pcall @@ -26,15 +27,20 @@ Luan.type = BasicLuan.type Luan.values = BasicLuan.values +local function load(text,source_name,env,persist) + return java_load( text, source_name or "load", persist==true, env ) +end +Luan.load = load + function Luan.do_file(uri) - local fn = Luan.load_file(uri) or error("file '"..uri.."' not found") + local fn = load_file(uri) or error("file '"..uri.."' not found") return fn() end Luan.VERSION = Luan.do_file "classpath:luan/version.luan" function Luan.eval(s,source_name,env) - return Luan.load( "return "..s, source_name or "eval", env )() + return load( "return "..s, source_name or "eval", env )() end return Luan
--- a/src/luan/modules/PackageLuan.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/modules/PackageLuan.java Sun Dec 29 15:25:07 2019 -0700 @@ -61,7 +61,7 @@ } catch(IOException e) { throw new RuntimeException(e); } - LuanFunction loader = luan.load(src,modName); + LuanFunction loader = luan.load(src,modName,true); mod = Luan.first( loader.call(modName) ); @@ -76,7 +76,7 @@ if( src == null ) { mod = Boolean.FALSE; } else { - LuanFunction loader = luan.load(src,modName); + LuanFunction loader = luan.load(src,modName,true); mod = Luan.first( loader.call(modName) );
--- a/src/luan/modules/http/LuanHandler.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/modules/http/LuanHandler.java Sun Dec 29 15:25:07 2019 -0700 @@ -190,7 +190,7 @@ LuanCloner cloner = new LuanCloner(LuanCloner.Type.COMPLETE); luan = (Luan)cloner.clone(currentLuan); } - luan.load(text,"<eval_in_root>",null).call(); + luan.load(text,"<eval_in_root>",false,null).call(); currentLuan = luan; }
--- a/src/luan/modules/lucene/PostgresBackup.java Tue Dec 24 17:57:47 2019 -0700 +++ b/src/luan/modules/lucene/PostgresBackup.java Sun Dec 29 15:25:07 2019 -0700 @@ -163,7 +163,7 @@ } private static Object eval(String s,LuanTable env) throws LuanException { - LuanFunction fn = env.luan().load( "return "+s, "PostgresBackup", env ); + LuanFunction fn = env.luan().load( "return "+s, "PostgresBackup", false, env ); return fn.call(); }
--- a/website/src/manual.html Tue Dec 24 17:57:47 2019 -0700 +++ b/website/src/manual.html Sun Dec 29 15:25:07 2019 -0700 @@ -1812,7 +1812,7 @@ -<h4 heading><a name="Luan.eval" href="#Luan.eval"><code>Luan.eval (text [, source_name] [, env])</code></a></h4> +<h4 heading><a name="Luan.eval" href="#Luan.eval"><code>Luan.eval (text [, source_name [, env]])</code></a></h4> <p> Evaluates <code>text</code> as a Luan expression. @@ -1876,7 +1876,7 @@ -<h4 heading><a name="Luan.load" href="#Luan.load"><code>Luan.load (text, source_name [, env])</code></a></h4> +<h4 heading><a name="Luan.load" href="#Luan.load"><code>Luan.load (text, [source_name [, env [, persist]]])</code></a></h4> <p> Loads a chunk. @@ -1888,33 +1888,21 @@ otherwise, throws an error. <p> -The <code>source_name</code> parameter is a string saying where the text came from. It is used to produce error messages. +The <code>source_name</code> parameter is a string saying where the text came from. It is used to produce error messages. Defaults to "load". <p> If the <code>env</code> parameter is supplied, it becomes the <code>_ENV</code> of the chunk. - -<h4 heading><a name="Luan.load_file" href="#Luan.load_file"><code>Luan.load_file ([file_uri])</code></a></h4> +<p> +The <code>persist</code> parameter is a boolean which determines if the compiled code is persistently cached to a temporary file. Defaults to <code>false</code>. + + +<h4 heading><a name="Luan.load_file" href="#Luan.load_file"><code>Luan.load_file (file_uri)</code></a></h4> <p> Similar to <a href="#Luan.load"><code>load</code></a>, -but gets the chunk from file <code>file_uri</code> -or from the standard input, -if no file uri is given. - -<p> -Could be defined as: - -<pre> - function Luan.load_file(file_uri) - file_uri = file_uri or "stdin:" - local f = Io.uri(file_uri) - if not f.exists() then - return nil - end - return <a href="#Luan.load">Luan.load</a>( f.read_text(), file_uri ) - end -</pre> +but gets the chunk from file <code>file_uri</code>. +<code>file_uri</code> can be a string or a uri table. <h4 heading><a name="Luan.new_error" href="#Luan.new_error"><code>Luan.new_error (message)</code></a></h4>