Mercurial Hosting > luan
diff lucene/src/luan/modules/lucene/LuceneIndex.java @ 544:c5a93767cc5c
lucene overhaul, untested
author | Franklin Schmidt <fschmidt@gmail.com> |
---|---|
date | Fri, 12 Jun 2015 19:11:44 -0600 |
parents | ef0336efe33c |
children | ddcd4296107a |
line wrap: on
line diff
--- a/lucene/src/luan/modules/lucene/LuceneIndex.java Mon Jun 08 01:11:08 2015 -0400 +++ b/lucene/src/luan/modules/lucene/LuceneIndex.java Fri Jun 12 19:11:44 2015 -0600 @@ -5,12 +5,13 @@ import java.io.FileOutputStream; import java.io.FileInputStream; import java.io.IOException; +import java.util.Iterator; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.zip.ZipOutputStream; import java.util.zip.ZipEntry; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -23,8 +24,15 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; -import org.apache.lucene.queryparser.flexible.core.QueryNodeException; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.IndexSearcher; +import sane.lucene.queryparser.SaneQueryParser; +import sane.lucene.queryparser.FieldParser; +import sane.lucene.queryparser.MultiFieldParser; +import sane.lucene.queryparser.StringFieldParser; +import sane.lucene.queryparser.NumberFieldParser; +import sane.lucene.queryparser.ParseException; import luan.modules.Utils; import luan.Luan; import luan.LuanState; @@ -32,6 +40,7 @@ import luan.LuanFunction; import luan.LuanJavaFunction; import luan.LuanException; +import luan.LuanMeta; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +49,8 @@ private static final Logger logger = LoggerFactory.getLogger(LuceneIndex.class); private static final String FLD_NEXT_ID = "nextId"; + private static final Analyzer analyzer = new KeywordAnalyzer(); + public static final FieldParser STRING_FIELD_PARSER = new StringFieldParser(analyzer); final Lock writeLock = new ReentrantLock(); private final File indexDir; @@ -47,15 +58,16 @@ final IndexWriter writer; private DirectoryReader reader; private LuceneSearcher searcher; - public final FieldTable fields = new FieldTable(); private boolean isClosed = false; + private final MultiFieldParser mfp = new MultiFieldParser(); public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException { + mfp.fields.put( "type", STRING_FIELD_PARSER ); + mfp.fields.put( "id", NumberFieldParser.LONG ); File indexDir = new File(indexDirStr); this.indexDir = indexDir; Directory dir = FSDirectory.open(indexDir); Version version = Version.LUCENE_4_9; - Analyzer analyzer = new StandardAnalyzer(version); IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); conf.setIndexDeletionPolicy(snapshotDeletionPolicy); @@ -68,20 +80,7 @@ } Document toLucene(LuanState luan,LuanTable table) throws LuanException { - return LuceneDocument.toLucene(luan,table,fields.map); - } - - LuanTable toTable(LuanState luan,Document doc) throws LuanException { - return LuceneDocument.toTable(luan,doc,fields.reverseMap); - } - - public String map_field_name(String fld) { - String s = fields.map.get(fld); - return s!=null ? s : fld; - } - - Term newTerm(String fld,String text) { - return new Term(map_field_name(fld),text); + return LuceneDocument.toLucene(luan,table,mfp.fields.keySet()); } public LuceneWriter openWriter() { @@ -120,13 +119,13 @@ private final int idBatch = 10; private void initId(LuanState luan) throws LuanException, IOException { - TopDocs td = searcher.search(new TermQuery(newTerm("type","next_id")),1); + IndexSearcher searcher = this.searcher.searcher; + TopDocs td = searcher.search(new TermQuery(new Term("type","next_id")),1); switch(td.totalHits) { case 0: break; // do nothing case 1: - LuanTable doc = searcher.doc(luan,td.scoreDocs[0].doc); - idLim = (Long)doc.rawGet(FLD_NEXT_ID); + idLim = (Long)searcher.doc(td.scoreDocs[0].doc).getField(FLD_NEXT_ID).numericValue(); id = idLim; break; default: @@ -134,16 +133,15 @@ } } - synchronized String nextId(LuanState luan) throws LuanException, IOException { - String rtn = Long.toString(++id); - if( id > idLim ) { + synchronized long nextId(LuanState luan) throws LuanException, IOException { + if( ++id > idLim ) { idLim += idBatch; LuanTable doc = new LuanTable(); doc.rawPut( "type", "next_id" ); doc.rawPut( FLD_NEXT_ID, idLim ); - writer.updateDocument(newTerm("type","next_id"),toLucene(luan,doc)); + writer.updateDocument(new Term("type","next_id"),toLucene(luan,doc)); } - return rtn; + return id; } @@ -210,10 +208,47 @@ } - public Query parse(String s) throws QueryNodeException { - StandardQueryParser qp = new StandardQueryParser(); - qp.setQueryNodeProcessor(new LuanQueryNodeProcessor(this,qp.getQueryNodeProcessor())); - return qp.parse(s,null); + + public final LuanMeta indexedFieldsMeta = new LuanMeta() { + + @Override public boolean canNewindex() { + return true; + } + + @Override public Object __index(LuanState luan,LuanTable tbl,Object key) { + return mfp.fields.get(key); + } + + @Override public void __new_index(LuanState luan,LuanTable tbl,Object key,Object value) throws LuanException { + if( !(key instanceof String) ) + throw luan.exception("key must be string"); + String field = (String)key; + if( value==null ) { // delete + mfp.fields.remove(field); + return; + } + if( !(value instanceof FieldParser) ) + throw luan.exception("value must be FieldParser like the values of Lucene.type"); + FieldParser parser = (FieldParser)value; + mfp.fields.put( field, parser ); + } + + @Override public final Iterator keys(LuanTable tbl) { + return mfp.fields.keySet().iterator(); + } + + @Override protected String type(LuanTable tbl) { + return "lucene-indexed-fields"; + } + + }; + + public Query parseQuery(String s) throws ParseException { + return SaneQueryParser.parseQuery(mfp,s); + } + + public Sort parseSort(String s) throws ParseException { + return SaneQueryParser.parseSort(mfp,s); } }