Mercurial Hosting > luan

comparison lucene/src/luan/modules/lucene/LuceneIndex.java @ 233:ef39bc4d3f70

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
basic lucene works git-svn-id: https://luan-java.googlecode.com/svn/trunk@234 21e917c8-12df-6dd8-5cb6-c86387c605b9
author fschmidt@gmail.com <fschmidt@gmail.com@21e917c8-12df-6dd8-5cb6-c86387c605b9>
date Thu, 02 Oct 2014 02:58:55 +0000
parents 9ce18106f95a
children 3896138955b1
comparison
equal deleted inserted replaced
232:9ce18106f95a 233:ef39bc4d3f70
8 import java.util.concurrent.locks.ReentrantLock; 8 import java.util.concurrent.locks.ReentrantLock;
9 import java.util.zip.ZipOutputStream; 9 import java.util.zip.ZipOutputStream;
10 import java.util.zip.ZipEntry; 10 import java.util.zip.ZipEntry;
11 import org.apache.lucene.analysis.Analyzer; 11 import org.apache.lucene.analysis.Analyzer;
12 import org.apache.lucene.analysis.standard.StandardAnalyzer; 12 import org.apache.lucene.analysis.standard.StandardAnalyzer;
13 import org.apache.lucene.document.Document;
13 import org.apache.lucene.index.IndexWriter; 14 import org.apache.lucene.index.IndexWriter;
14 import org.apache.lucene.index.IndexWriterConfig; 15 import org.apache.lucene.index.IndexWriterConfig;
15 import org.apache.lucene.index.DirectoryReader; 16 import org.apache.lucene.index.DirectoryReader;
16 import org.apache.lucene.index.Term; 17 import org.apache.lucene.index.Term;
17 import org.apache.lucene.index.SnapshotDeletionPolicy; 18 import org.apache.lucene.index.SnapshotDeletionPolicy;
28 import luan.LuanJavaFunction; 29 import luan.LuanJavaFunction;
29 import luan.LuanException; 30 import luan.LuanException;
30 31
31 32
32 public final class LuceneIndex { 33 public final class LuceneIndex {
33 private static final String FLD_TYPE = LuceneWriter.FLD_TYPE;
34 private static final String FLD_NEXT_ID = "nextId"; 34 private static final String FLD_NEXT_ID = "nextId";
35 35
36 final Lock writeLock = new ReentrantLock(); 36 final Lock writeLock = new ReentrantLock();
37 private final File indexDir; 37 private final File indexDir;
38 final SnapshotDeletionPolicy snapshotDeletionPolicy; 38 final SnapshotDeletionPolicy snapshotDeletionPolicy;
39 final IndexWriter writer; 39 final IndexWriter writer;
40 private DirectoryReader reader; 40 private DirectoryReader reader;
41 private LuceneSearcher searcher; 41 private LuceneSearcher searcher;
42 final FieldTable fields = new FieldTable();
42 43
43 public LuceneIndex(String indexDirStr) { 44 public LuceneIndex(LuanState luan,String indexDirStr) throws LuanException, IOException {
44 try { 45 File indexDir = new File(indexDirStr);
45 File indexDir = new File(indexDirStr); 46 this.indexDir = indexDir;
46 this.indexDir = indexDir; 47 Directory dir = FSDirectory.open(indexDir);
47 Directory dir = FSDirectory.open(indexDir); 48 Version version = Version.LUCENE_4_9;
48 Version version = Version.LUCENE_4_9; 49 Analyzer analyzer = new StandardAnalyzer(version);
49 Analyzer analyzer = new StandardAnalyzer(version); 50 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer);
50 IndexWriterConfig conf = new IndexWriterConfig(version,analyzer); 51 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
51 snapshotDeletionPolicy = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()); 52 conf.setIndexDeletionPolicy(snapshotDeletionPolicy);
52 conf.setIndexDeletionPolicy(snapshotDeletionPolicy); 53 writer = new IndexWriter(dir,conf);
53 writer = new IndexWriter(dir,conf); 54 writer.commit(); // commit index creation
54 writer.commit(); // commit index creation 55 reader = DirectoryReader.open(dir);
55 reader = DirectoryReader.open(dir); 56 searcher = new LuceneSearcher(this,reader);
56 searcher = new LuceneSearcher(reader); 57 initId(luan);
57 initId(); 58 }
58 } catch(IOException e) { 59
59 throw new RuntimeException(e); 60 Document toLucene(LuanState luan,LuanTable table) throws LuanException {
60 } 61 return LuceneDocument.toLucene(luan,table,fields.map);
62 }
63
64 LuanTable toTable(LuanState luan,Document doc) throws LuanException {
65 return LuceneDocument.toTable(luan,doc,fields.reverseMap);
66 }
67
68 String fixFieldName(String fld) {
69 String s = fields.map.get(fld);
70 return s!=null ? s : fld;
71 }
72
73 Term newTerm(String fld,String text) {
74 return new Term(fixFieldName(fld),text);
61 } 75 }
62 76
63 public LuceneWriter openWriter() { 77 public LuceneWriter openWriter() {
64 return new LuceneWriter(this); 78 return new LuceneWriter(this);
65 } 79 }
66 80
67 public synchronized LuceneSearcher openSearcher() { 81 synchronized LuceneSearcher openSearcher() throws IOException {
68 try { 82 DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
69 DirectoryReader newReader = DirectoryReader.openIfChanged(reader); 83 if( newReader != null ) {
70 if( newReader != null ) { 84 reader.decRef();
71 reader.decRef(); 85 reader = newReader;
72 reader = newReader; 86 searcher = new LuceneSearcher(this,reader);
73 searcher = new LuceneSearcher(reader);
74 }
75 reader.incRef();
76 return searcher;
77 } catch(IOException e) {
78 throw new RuntimeException(e);
79 } 87 }
88 reader.incRef();
89 return searcher;
80 } 90 }
81 91
82 public LuceneSnapshot openSnapshot() { 92 LuceneSnapshot openSnapshot() throws IOException {
83 return new LuceneSnapshot(this); 93 return new LuceneSnapshot(this);
84 } 94 }
85 95
86 96
87 private long id = 0; 97 private long id = 0;
88 private long idLim = 0; 98 private long idLim = 0;
89 private final int idBatch = 10; 99 private final int idBatch = 10;
90 100
91 private void initId() throws IOException { 101 private void initId(LuanState luan) throws LuanException, IOException {
92 TopDocs td = searcher.search(new TermQuery(new Term(FLD_TYPE,"next_id")),1); 102 TopDocs td = searcher.search(new TermQuery(newTerm("type","next_id")),1);
93 switch(td.totalHits) { 103 switch(td.totalHits) {
94 case 0: 104 case 0:
95 break; // do nothing 105 break; // do nothing
96 case 1: 106 case 1:
97 LuanTable doc = searcher.doc(td.scoreDocs[0].doc); 107 LuanTable doc = searcher.doc(luan,td.scoreDocs[0].doc);
98 idLim = (Long)doc.get(FLD_NEXT_ID); 108 idLim = (Long)doc.get(FLD_NEXT_ID);
99 id = idLim; 109 id = idLim;
100 break; 110 break;
101 default: 111 default:
102 throw new RuntimeException(); 112 throw new RuntimeException();
103 } 113 }
104 } 114 }
105 115
106 synchronized String nextId() { 116 synchronized String nextId(LuanState luan) throws LuanException, IOException {
107 try { 117 String rtn = Long.toString(++id);
108 String rtn = Long.toString(++id); 118 if( id > idLim ) {
109 if( id > idLim ) { 119 idLim += idBatch;
110 idLim += idBatch; 120 LuanTable doc = Luan.newTable();
111 LuanTable doc = Luan.newTable(); 121 doc.put( "type", "next_id" );
112 doc.put( FLD_TYPE, "next_id" ); 122 doc.put( FLD_NEXT_ID, idLim );
113 doc.put( FLD_NEXT_ID, idLim ); 123 writer.updateDocument(newTerm("type","next_id"),toLucene(luan,doc));
114 writer.updateDocument(new Term(FLD_TYPE,"next_id"),LuceneDocument.toLucene(doc));
115 }
116 return rtn;
117 } catch(IOException e) {
118 throw new RuntimeException(e);
119 } 124 }
125 return rtn;
120 } 126 }
121 127
122 128
123 public void backup(String zipFile) { 129 public void backup(LuanState luan,String zipFile) throws LuanException, IOException {
124 if( !zipFile.endsWith(".zip") ) 130 if( !zipFile.endsWith(".zip") )
125 throw new RuntimeException("file "+zipFile+" doesn't end with '.zip'"); 131 throw luan.exception("file "+zipFile+" doesn't end with '.zip'");
126 LuceneSnapshot snapshot = openSnapshot(); 132 LuceneSnapshot snapshot = openSnapshot();
127 try { 133 try {
128 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile)); 134 ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile));
129 for( String fileName : snapshot.getFileNames() ) { 135 for( String fileName : snapshot.getFileNames() ) {
130 out.putNextEntry(new ZipEntry(fileName)); 136 out.putNextEntry(new ZipEntry(fileName));
132 Utils.copyAll(in,out); 138 Utils.copyAll(in,out);
133 in.close(); 139 in.close();
134 out.closeEntry(); 140 out.closeEntry();
135 } 141 }
136 out.close(); 142 out.close();
137 } catch(IOException e) {
138 throw new RuntimeException(e);
139 } finally { 143 } finally {
140 snapshot.close(); 144 snapshot.close();
141 } 145 }
142 } 146 }
143 147
157 } finally { 161 } finally {
158 writer.close(); 162 writer.close();
159 } 163 }
160 } 164 }
161 165
162 public void Searcher(LuanState luan,LuanFunction fn) throws LuanException, IOException { 166 public Object Searcher(LuanState luan,LuanFunction fn) throws LuanException, IOException {
163 LuceneSearcher searcher = openSearcher(); 167 LuceneSearcher searcher = openSearcher();
164 try { 168 try {
165 luan.call( fn, new Object[]{searcher.table()} ); 169 return luan.call( fn, new Object[]{searcher.table()} );
166 } finally { 170 } finally {
167 searcher.close(); 171 searcher.close();
168 } 172 }
169 } 173 }
170 174
173 } 177 }
174 178
175 public LuanTable table() { 179 public LuanTable table() {
176 LuanTable tbl = Luan.newTable(); 180 LuanTable tbl = Luan.newTable();
177 try { 181 try {
182 tbl.put("fields",fields);
178 add( tbl, "to_string" ); 183 add( tbl, "to_string" );
179 add( tbl, "backup", String.class ); 184 add( tbl, "backup", LuanState.class, String.class );
180 add( tbl, "Writer", LuanState.class, LuanFunction.class ); 185 add( tbl, "Writer", LuanState.class, LuanFunction.class );
181 add( tbl, "Searcher", LuanState.class, LuanFunction.class ); 186 add( tbl, "Searcher", LuanState.class, LuanFunction.class );
182 } catch(NoSuchMethodException e) { 187 } catch(NoSuchMethodException e) {
183 throw new RuntimeException(e); 188 throw new RuntimeException(e);
184 } 189 }