Mercurial Hosting > luan
changeset 233:ef39bc4d3f70
basic lucene works
git-svn-id: https://luan-java.googlecode.com/svn/trunk@234 21e917c8-12df-6dd8-5cb6-c86387c605b9
author | fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9> |
---|---|
date | Thu, 02 Oct 2014 02:58:55 +0000 |
parents | 9ce18106f95a |
children | b25feac318d8 |
files | core/src/luan/AbstractLuanTable.java core/src/luan/LuanTable.java core/src/luan/LuanTableImpl.java core/src/luan/init.luan core/src/luan/modules/BasicLuan.java lucene/src/luan/modules/lucene/FieldTable.java lucene/src/luan/modules/lucene/Lucene.luan lucene/src/luan/modules/lucene/LuceneDocument.java lucene/src/luan/modules/lucene/LuceneIndex.java lucene/src/luan/modules/lucene/LuceneSearcher.java lucene/src/luan/modules/lucene/LuceneSnapshot.java lucene/src/luan/modules/lucene/LuceneWriter.java |
diffstat | 12 files changed, 206 insertions(+), 138 deletions(-) [+] |
line wrap: on
line diff
--- a/core/src/luan/AbstractLuanTable.java Wed Oct 01 06:55:14 2014 +0000 +++ b/core/src/luan/AbstractLuanTable.java Thu Oct 02 02:58:55 2014 +0000 @@ -16,6 +16,14 @@ public abstract class AbstractLuanTable implements LuanTable { + @Override public boolean isEmpty() { + return isList() && length()==0; + } + + @Override public boolean isList() { + return asList().size() == asMap().size(); + } + @Override public List<Object> asList() { return Collections.emptyList(); }
--- a/core/src/luan/LuanTable.java Wed Oct 01 06:55:14 2014 +0000 +++ b/core/src/luan/LuanTable.java Thu Oct 02 02:58:55 2014 +0000 @@ -6,6 +6,7 @@ public interface LuanTable extends Iterable<Map.Entry<Object,Object>> { + public boolean isEmpty(); public boolean isList(); public List<Object> asList(); public Map<Object,Object> asMap();
--- a/core/src/luan/LuanTableImpl.java Wed Oct 01 06:55:14 2014 +0000 +++ b/core/src/luan/LuanTableImpl.java Thu Oct 02 02:58:55 2014 +0000 @@ -354,8 +354,4 @@ @Override public void setMetatable(LuanTable metatable) { this.metatable = metatable; } - - public boolean isEmpty() { - return (list==null || list.isEmpty()) && (map==null || map.isEmpty()); - } }
--- a/core/src/luan/init.luan Wed Oct 01 06:55:14 2014 +0000 +++ b/core/src/luan/init.luan Thu Oct 02 02:58:55 2014 +0000 @@ -18,6 +18,7 @@ Package.global(Basic,"assert_number") Package.global(Basic,"assert_string") Package.global(Basic,"assert_table") +Package.global(Basic,"assert_integer") local do_file = Package.global(Basic,"do_file") Package.global(Basic,"error") Package.global(Basic,"get_metatable")
--- a/core/src/luan/modules/BasicLuan.java Wed Oct 01 06:55:14 2014 +0000 +++ b/core/src/luan/modules/BasicLuan.java Thu Oct 02 02:58:55 2014 +0000 @@ -30,6 +30,7 @@ add( module, "assert_number", LuanState.class, Number.class ); add( module, "assert_string", LuanState.class, String.class ); add( module, "assert_table", LuanState.class, LuanTable.class ); + add( module, "assert_integer", LuanState.class, Integer.TYPE ); add( module, "do_file", LuanState.class, String.class ); add( module, "error", LuanState.class, Object.class ); add( module, "get_metatable", LuanState.class, Object.class ); @@ -194,6 +195,10 @@ return v; } + public static int assert_integer(LuanState luan,int v) throws LuanException { + return v; + } + public static String repr(LuanState luan,Object v) throws LuanException { return luan.repr(v); }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lucene/src/luan/modules/lucene/FieldTable.java Thu Oct 02 02:58:55 2014 +0000 @@ -0,0 +1,45 @@ +package luan.modules.lucene; + +import java.util.Map; +import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.Iterator; +import luan.AbstractLuanTable; + + +class FieldTable extends AbstractLuanTable { + final Map<String,String> map = new ConcurrentHashMap<String,String>(); + final Map<String,String> reverseMap = new ConcurrentHashMap<String,String>(); + + FieldTable() { + put("type","type index"); + put("id","id index"); + } + + @Override public void put(Object key,Object value) { + if( !(key instanceof String) ) + throw new UnsupportedOperationException("key must be string"); + String name = (String)key; + if( value==null ) { // delete + reverseMap.remove(map.remove(name)); + return; + } + if( !(value instanceof String) ) + throw new UnsupportedOperationException("value must be string"); + String field = (String)value; + map.put(name,field); + reverseMap.put(field,name); + } + + @Override public final Object get(Object key) { + return map.get(key); + } + + @Override public final Iterator<Map.Entry<Object,Object>> iterator() { + return new HashMap<Object,Object>(map).entrySet().iterator(); + } + + @Override protected String type() { + return "lucene-field-table"; + } +}
--- a/lucene/src/luan/modules/lucene/Lucene.luan Wed Oct 01 06:55:14 2014 +0000 +++ b/lucene/src/luan/modules/lucene/Lucene.luan Thu Oct 02 02:58:55 2014 +0000 @@ -1,10 +1,6 @@ import "Java" import "luan.modules.lucene.LuceneIndex" -standard_fields = { - type = "type index"; - id = "id index"; -} function Index(indexDir) local index = LuceneIndex.new(indexDir).table() @@ -22,12 +18,12 @@ end function index.get_document(query) - index.Searcher( function(searcher) + return index.Searcher( function(searcher) local results, _, total_hits = searcher.search(query,1) if total_hits == 0 then return nil elseif total_hits > 1 then - error "found " .. total_hits .. " documents" + error( "found " .. total_hits .. " documents" ) end return results() end )
--- a/lucene/src/luan/modules/lucene/LuceneDocument.java Wed Oct 01 06:55:14 2014 +0000 +++ b/lucene/src/luan/modules/lucene/LuceneDocument.java Thu Oct 02 02:58:55 2014 +0000 @@ -12,9 +12,12 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.IntField; import org.apache.lucene.document.LongField; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.util.BytesRef; import luan.Luan; +import luan.LuanState; import luan.LuanTable; +import luan.LuanException; public class LuceneDocument { @@ -23,16 +26,17 @@ private LuceneDocument(String a) {} // never - static Document toLucene(LuanTable table) { + static Document toLucene(LuanState luan,LuanTable table,Map<String,String> nameMap) throws LuanException { Document doc = new Document(); for( Map.Entry<Object,Object> entry : table ) { Object key = entry.getKey(); if( !(key instanceof String) ) - throw new IllegalArgumentException("key must be string"); + throw luan.exception("key must be string"); String name = (String)key; Object value = entry.getValue(); - if( value == null ) - continue; + String newName = nameMap.get(name); + if( newName != null ) + name = newName; Set<String> flags = new HashSet<String>(); String[] a = name.split(" +"); for( int i=1; i<a.length; i++ ) { @@ -59,23 +63,33 @@ } else { doc.add(new StoredField(name, i)); } + } else if( value instanceof Double ) { + double i = (Double)value; + if( flags.remove(INDEX) ) { + doc.add(new DoubleField(name, i, Field.Store.YES)); + } else { + doc.add(new StoredField(name, i)); + } } else if( value instanceof byte[] ) { byte[] b = (byte[])value; doc.add(new StoredField(name, b)); } else - throw new IllegalArgumentException("invalid value type "+value.getClass()+"' for '"+name+"'"); + throw luan.exception("invalid value type "+value.getClass()+"' for '"+name+"'"); if( !flags.isEmpty() ) - throw new IllegalArgumentException("invalid flags "+flags+" in '"+name+"'"); + throw luan.exception("invalid flags "+flags+" in '"+name+"'"); } return doc; } - static LuanTable toTable(Document doc) { + static LuanTable toTable(LuanState luan,Document doc,Map<String,String> nameMap) throws LuanException { if( doc==null ) return null; LuanTable table = Luan.newTable(); for( IndexableField ifld : doc ) { String name = ifld.name(); + String newName = nameMap.get(name); + if( newName != null ) + name = newName; BytesRef br = ifld.binaryValue(); if( br != null ) { table.put(name,br.bytes); @@ -91,7 +105,7 @@ table.put(name,s); continue; } - throw new RuntimeException("invalid field type for "+ifld); + throw luan.exception("invalid field type for "+ifld); } return table; }
--- a/lucene/src/luan/modules/lucene/LuceneIndex.java Wed Oct 01 06:55:14 2014 +0000 +++ b/lucene/src/luan/modules/lucene/LuceneIndex.java Thu Oct 02 02:58:55 2014 +0000 @@ -10,6 +10,7 @@ import java.util.zip.ZipEntry; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.DirectoryReader; @@ -30,7 +31,6 @@ public final class LuceneIndex { - private static final String FLD_TYPE = LuceneWriter.FLD_TYPE; private static final String FLD_NEXT_ID = "nextId"; final Lock writeLock = new ReentrantLock(); @@ -39,47 +39,57 @@ final IndexWriter writer; private DirectoryReader reader; private LuceneSearcher searcher; + final FieldTable fields = new FieldTable(); - public LuceneIndex(String indexDirStr) { - try { - File indexDir = new File(indexDirStr); - this.indexDir = indexDir; - Directory dir = FSDirectory.open(indexDir); - Version version = Version.LUCENE_4_9; - Analyzer analyzer = new StandardAnalyzer(version); - IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); - snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); - conf.setIndexDeletionPolicy(snapshotDeletionPolicy); - writer = new IndexWriter(dir,conf); - writer.commit(); // commit index creation - reader = DirectoryReader.open(dir); - searcher = new LuceneSearcher(reader); - initId(); - } catch(IOException e) { - throw new RuntimeException(e); - } + public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException { + File indexDir = new File(indexDirStr); + this.indexDir = indexDir; + Directory dir = FSDirectory.open(indexDir); + Version version = Version.LUCENE_4_9; + Analyzer analyzer = new StandardAnalyzer(version); + IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); + snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); + conf.setIndexDeletionPolicy(snapshotDeletionPolicy); + writer = new IndexWriter(dir,conf); + writer.commit(); // commit index creation + reader = DirectoryReader.open(dir); + searcher = new LuceneSearcher(this,reader); + initId(luan); + } + + Document toLucene(LuanState luan,LuanTable table) throws LuanException { + return LuceneDocument.toLucene(luan,table,fields.map); + } + + LuanTable toTable(LuanState luan,Document doc) throws LuanException { + return LuceneDocument.toTable(luan,doc,fields.reverseMap); + } + + String fixFieldName(String fld) { + String s = fields.map.get(fld); + return s!=null ? s : fld; + } + + Term newTerm(String fld,String text) { + return new Term(fixFieldName(fld),text); } public LuceneWriter openWriter() { return new LuceneWriter(this); } - public synchronized LuceneSearcher openSearcher() { - try { - DirectoryReader newReader = DirectoryReader.openIfChanged(reader); - if( newReader != null ) { - reader.decRef(); - reader = newReader; - searcher = new LuceneSearcher(reader); - } - reader.incRef(); - return searcher; - } catch(IOException e) { - throw new RuntimeException(e); + synchronized LuceneSearcher openSearcher() throws IOException { + DirectoryReader newReader = DirectoryReader.openIfChanged(reader); + if( newReader != null ) { + reader.decRef(); + reader = newReader; + searcher = new LuceneSearcher(this,reader); } + reader.incRef(); + return searcher; } - public LuceneSnapshot openSnapshot() { + LuceneSnapshot openSnapshot() throws IOException { return new LuceneSnapshot(this); } @@ -88,13 +98,13 @@ private long idLim = 0; private final int idBatch = 10; - private void initId() throws IOException { - TopDocs td = searcher.search(new TermQuery(new Term(FLD_TYPE,"next_id")),1); + private void initId(LuanState luan) throws LuanException, IOException { + TopDocs td = searcher.search(new TermQuery(newTerm("type","next_id")),1); switch(td.totalHits) { case 0: break; // do nothing case 1: - LuanTable doc = searcher.doc(td.scoreDocs[0].doc); + LuanTable doc = searcher.doc(luan,td.scoreDocs[0].doc); idLim = (Long)doc.get(FLD_NEXT_ID); id = idLim; break; @@ -103,26 +113,22 @@ } } - synchronized String nextId() { - try { - String rtn = Long.toString(++id); - if( id > idLim ) { - idLim += idBatch; - LuanTable doc = Luan.newTable(); - doc.put( FLD_TYPE, "next_id" ); - doc.put( FLD_NEXT_ID, idLim ); - writer.updateDocument(new Term(FLD_TYPE,"next_id"),LuceneDocument.toLucene(doc)); - } - return rtn; - } catch(IOException e) { - throw new RuntimeException(e); + synchronized String nextId(LuanState luan) throws LuanException, IOException { + String rtn = Long.toString(++id); + if( id > idLim ) { + idLim += idBatch; + LuanTable doc = Luan.newTable(); + doc.put( "type", "next_id" ); + doc.put( FLD_NEXT_ID, idLim ); + writer.updateDocument(newTerm("type","next_id"),toLucene(luan,doc)); } + return rtn; } - public void backup(String zipFile) { + public void backup(LuanState luan,String zipFile) throws LuanException, IOException { if( !zipFile.endsWith(".zip") ) - throw new RuntimeException("file "+zipFile+" doesn't end with '.zip'"); + throw luan.exception("file "+zipFile+" doesn't end with '.zip'"); LuceneSnapshot snapshot = openSnapshot(); try { ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile)); @@ -134,8 +140,6 @@ out.closeEntry(); } out.close(); - } catch(IOException e) { - throw new RuntimeException(e); } finally { snapshot.close(); } @@ -159,10 +163,10 @@ } } - public void Searcher(LuanState luan,LuanFunction fn) throws LuanException, IOException { + public Object Searcher(LuanState luan,LuanFunction fn) throws LuanException, IOException { LuceneSearcher searcher = openSearcher(); try { - luan.call( fn, new Object[]{searcher.table()} ); + return luan.call( fn, new Object[]{searcher.table()} ); } finally { searcher.close(); } @@ -175,8 +179,9 @@ public LuanTable table() { LuanTable tbl = Luan.newTable(); try { + tbl.put("fields",fields); add( tbl, "to_string" ); - add( tbl, "backup", String.class ); + add( tbl, "backup", LuanState.class, String.class ); add( tbl, "Writer", LuanState.class, LuanFunction.class ); add( tbl, "Searcher", LuanState.class, LuanFunction.class ); } catch(NoSuchMethodException e) {
--- a/lucene/src/luan/modules/lucene/LuceneSearcher.java Wed Oct 01 06:55:14 2014 +0000 +++ b/lucene/src/luan/modules/lucene/LuceneSearcher.java Thu Oct 02 02:58:55 2014 +0000 @@ -25,23 +25,21 @@ public final class LuceneSearcher { + private final LuceneIndex index; private final IndexSearcher searcher; - LuceneSearcher(IndexReader reader) { + LuceneSearcher(LuceneIndex index,IndexReader reader) { + this.index = index; this.searcher = new IndexSearcher(reader); } // call in finally block - void close() { - try { - searcher.getIndexReader().decRef(); - } catch(IOException e) { - throw new RuntimeException(e); - } + void close() throws IOException { + searcher.getIndexReader().decRef(); } - public LuanTable doc(int docID) throws IOException { - return LuceneDocument.toTable(searcher.doc(docID)); + LuanTable doc(LuanState luan,int docID) throws LuanException, IOException { + return index.toTable(luan,searcher.doc(docID)); } TopDocs search(Query query,int n) throws IOException { @@ -54,46 +52,57 @@ // luan - private Query termQuery(Map<Object,Object> map) { + private Query termQuery(LuanTable queryTbl) { + if( queryTbl.length() != 0 ) + return null; + Map<Object,Object> map = queryTbl.asMap(); if( map.size() != 1 ) return null; Map.Entry<Object,Object> entry = map.entrySet().iterator().next(); Object key = entry.getKey(); Object value = entry.getValue(); if( key instanceof String && value instanceof String ) { - return new TermQuery(new Term( (String)key, (String)value )); + return new TermQuery(index.newTerm( (String)key, (String)value )); } return null; } - private Query booleanQuery(Map<Object,Object> map) { + private Query booleanQuery(LuanTable queryTbl) { + if( !queryTbl.isList() ) + return null; + List<Object> clauses = queryTbl.asList(); BooleanQuery query = new BooleanQuery(); - for( Map.Entry<Object,Object> entry : map.entrySet() ) { - Object key = entry.getKey(); - Object value = entry.getValue(); - if( !(key instanceof String && value instanceof LuanTable) ) + for( Object obj : clauses ) { + if( !(obj instanceof LuanTable) ) return null; - Query subQuery = query( (LuanTable)value ); - if( subQuery == null ) + LuanTable tbl = (LuanTable)obj; + if( !(tbl.isList() && tbl.length()==2) ) + return null; + List<Object> list = tbl.asList(); + Object obj0 = list.get(0); + Object obj1 = list.get(1); + if( !(obj0 instanceof String && obj1 instanceof LuanTable) ) return null; BooleanClause.Occur occur; try { - occur = BooleanClause.Occur.valueOf( ((String)key).toUpperCase() ); + occur = BooleanClause.Occur.valueOf( ((String)obj0).toUpperCase() ); } catch(IllegalArgumentException e) { return null; } + Query subQuery = query( (LuanTable)obj1 ); + if( subQuery == null ) + return null; query.add(subQuery,occur); } return query; } private Query query(LuanTable queryTbl) { - Map<Object,Object> map = queryTbl.asMap(); - if( map.isEmpty() ) + if( queryTbl.isEmpty() ) return null; Query query; - query = termQuery(map); if(query!=null) return query; - query = booleanQuery(map); if(query!=null) return query; + query = termQuery(queryTbl); if(query!=null) return query; + query = booleanQuery(queryTbl); if(query!=null) return query; return null; } @@ -106,6 +115,7 @@ if( !(obj0 instanceof String && obj1 instanceof String) ) throw luan.exception("invalid sort field"+pos); String field = (String)obj0; + field = index.fixFieldName(field); SortField.Type type; try { type = SortField.Type.valueOf( ((String)obj1).toUpperCase() ); @@ -115,10 +125,12 @@ if( size == 2 ) return new SortField(field,type); Object obj2 = list.get(2); - if( !(obj2 instanceof Boolean) ) - throw luan.exception("invalid sort field"+pos+", 'reverse' must be boolean"); - boolean reverse = (Boolean)obj2; - return new SortField(field,type,reverse); + if( !(obj2 instanceof String) ) + throw luan.exception("invalid sort field"+pos+", order must be 'ascending' or 'descending'"); + String order = (String)obj2; + if( !(order.equals("ascending") || order.equals("descending")) ) + throw luan.exception("invalid sort field"+pos+", order must be 'ascending' or 'descending'"); + return new SortField( field, type, order.equals("descending") ); } private Sort sort(LuanState luan,LuanTable sortTbl) throws LuanException { @@ -155,7 +167,7 @@ if( i >= scoreDocs.length ) return LuanFunction.NOTHING; try { - LuanTable doc = doc(scoreDocs[i++].doc); + LuanTable doc = doc(luan,scoreDocs[i++].doc); return doc; } catch(IOException e) { throw luan.exception(e);
--- a/lucene/src/luan/modules/lucene/LuceneSnapshot.java Wed Oct 01 06:55:14 2014 +0000 +++ b/lucene/src/luan/modules/lucene/LuceneSnapshot.java Thu Oct 02 02:58:55 2014 +0000 @@ -9,30 +9,18 @@ private final LuceneIndex index; private final IndexCommit ic; - LuceneSnapshot(LuceneIndex index) { + LuceneSnapshot(LuceneIndex index) throws IOException { this.index = index; - try { - this.ic = index.snapshotDeletionPolicy.snapshot(); - } catch(IOException e) { - throw new RuntimeException(e); - } + this.ic = index.snapshotDeletionPolicy.snapshot(); } // call in finally block - public void close() { - try { - index.snapshotDeletionPolicy.release(ic); - } catch(IOException e) { - throw new RuntimeException(e); - } + public void close() throws IOException { + index.snapshotDeletionPolicy.release(ic); } - public Collection<String> getFileNames() { - try { - return ic.getFileNames(); - } catch(IOException e) { - throw new RuntimeException(e); - } + public Collection<String> getFileNames() throws IOException { + return ic.getFileNames(); } }
--- a/lucene/src/luan/modules/lucene/LuceneWriter.java Wed Oct 01 06:55:14 2014 +0000 +++ b/lucene/src/luan/modules/lucene/LuceneWriter.java Thu Oct 02 02:58:55 2014 +0000 @@ -15,9 +15,6 @@ public final class LuceneWriter { - public static final String FLD_TYPE = "type index"; - public static final String FLD_ID = "id index"; - private final LuceneIndex index; LuceneWriter(LuceneIndex index) { @@ -34,12 +31,12 @@ index.writer.commit(); } - void addDocument(LuanTable doc) throws IOException { - index.writer.addDocument(LuceneDocument.toLucene(doc)); + void addDocument(LuanState luan,LuanTable doc) throws LuanException, IOException { + index.writer.addDocument(index.toLucene(luan,doc)); } - void updateDocument(Term term,LuanTable doc) throws IOException { - index.writer.updateDocument(term,LuceneDocument.toLucene(doc)); + void updateDocument(LuanState luan,Term term,LuanTable doc) throws LuanException, IOException { + index.writer.updateDocument(term,index.toLucene(luan,doc)); } public void delete_documents(LuanState luan,LuanTable tblTerms) throws LuanException, IOException { @@ -51,25 +48,25 @@ throw luan.exception("key must be a string but got "+key.getClass().getSimpleName()); if( !(value instanceof String) ) throw luan.exception("value must be a string but got "+value.getClass().getSimpleName()); - list.add( new Term( (String)key, (String)value ) ); + list.add( index.newTerm( (String)key, (String)value ) ); } index.writer.deleteDocuments(list.toArray(new Term[list.size()])); } - String nextId() { - return index.nextId(); + String nextId(LuanState luan) throws LuanException, IOException { + return index.nextId(luan); } - public void save_document(LuanTable doc) throws IOException { - if( doc.get(FLD_TYPE)==null ) - throw new RuntimeException("missing '"+FLD_TYPE+"'"); - String id = (String)doc.get(FLD_ID); + public void save_document(LuanState luan,LuanTable doc) throws LuanException, IOException { + if( doc.get("type")==null ) + throw luan.exception("missing 'type'"); + String id = (String)doc.get("id"); if( id == null ) { - id = nextId(); - doc.put(FLD_ID,id); - addDocument(doc); + id = nextId(luan); + doc.put("id",id); + addDocument(luan,doc); } else { - updateDocument(new Term(FLD_ID,id),doc); + updateDocument(luan,index.newTerm("id",id),doc); } } @@ -82,7 +79,7 @@ LuanTable table() { LuanTable tbl = Luan.newTable(); try { - add( tbl, "save_document", LuanTable.class ); + add( tbl, "save_document", LuanState.class, LuanTable.class ); add( tbl, "delete_documents", LuanState.class, LuanTable.class ); } catch(NoSuchMethodException e) { throw new RuntimeException(e);