Mercurial Hosting > luan
annotate src/goodjava/lucene/api/LuceneIndexWriter.java @ 1544:35601f15ecc3
add lucene log tag and restore_from_log
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Sun, 20 Sep 2020 20:36:55 -0600 |
| parents | c27dc6af87ca |
| children | f48db13ae2d9 |
| rev | line source |
|---|---|
| 1460 | 1 package goodjava.lucene.api; |
| 2 | |
| 3 import java.io.IOException; | |
| 4 import java.util.Map; | |
| 5 import java.util.HashMap; | |
| 6 import java.util.List; | |
| 7 import org.apache.lucene.analysis.Analyzer; | |
| 8 import org.apache.lucene.document.Document; | |
| 9 import org.apache.lucene.document.Field; | |
| 10 import org.apache.lucene.document.StoredField; | |
| 11 import org.apache.lucene.document.StringField; | |
| 12 import org.apache.lucene.document.TextField; | |
| 13 import org.apache.lucene.document.IntField; | |
| 14 import org.apache.lucene.document.LongField; | |
| 15 import org.apache.lucene.document.DoubleField; | |
| 16 import org.apache.lucene.document.FloatField; | |
| 17 import org.apache.lucene.index.IndexWriter; | |
| 18 import org.apache.lucene.index.IndexWriterConfig; | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
19 import org.apache.lucene.index.LiveIndexWriterConfig; |
| 1460 | 20 import org.apache.lucene.index.Term; |
| 21 import org.apache.lucene.index.DirectoryReader; | |
| 22 import org.apache.lucene.index.IndexReader; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
23 import org.apache.lucene.index.CheckIndex; |
| 1460 | 24 import org.apache.lucene.search.Query; |
| 25 import org.apache.lucene.search.IndexSearcher; | |
| 26 import org.apache.lucene.store.Directory; | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
27 import goodjava.logging.Logger; |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
28 import goodjava.logging.LoggerFactory; |
| 1460 | 29 |
| 30 | |
| 31 public final class LuceneIndexWriter implements GoodIndexWriter { | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
32 private static final Logger logger = LoggerFactory.getLogger(LuceneIndexWriter.class); |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
33 private final FieldAnalyzer fieldAnalyzer; |
| 1465 | 34 public final IndexWriter luceneWriter; |
| 1460 | 35 public final GoodIndexWriterConfig goodConfig; |
| 36 private final Map<String,Boolean> indexedMap = new HashMap<String,Boolean>(); | |
| 37 | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
38 public LuceneIndexWriter(Directory dir,GoodIndexWriterConfig goodConfig) throws IOException { |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
39 IndexWriterConfig luceneConfig = goodConfig.newLuceneConfig(); |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
40 Analyzer analyzer = luceneConfig.getAnalyzer(); |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
41 if( !(analyzer instanceof FieldAnalyzer) ) |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
42 throw new RuntimeException("analyzer must be FieldAnalyzer"); |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
43 this.fieldAnalyzer = (FieldAnalyzer)analyzer; |
| 1465 | 44 this.luceneWriter = new IndexWriter(dir,luceneConfig); |
| 1460 | 45 this.goodConfig = goodConfig; |
| 1465 | 46 luceneWriter.commit(); // commit index creation |
| 1460 | 47 } |
| 48 | |
| 1539 | 49 public IndexWriter getLuceneIndexWriter() { |
| 50 return luceneWriter; | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
51 } |
|
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
52 |
| 1460 | 53 public void close() throws IOException { |
| 54 luceneWriter.close(); | |
| 55 } | |
| 56 | |
| 57 public void commit() throws IOException { | |
| 58 luceneWriter.commit(); | |
| 59 } | |
| 60 | |
| 61 public void rollback() throws IOException { | |
| 62 luceneWriter.rollback(); | |
| 63 } | |
| 64 | |
| 65 public void deleteAll() throws IOException { | |
| 66 luceneWriter.deleteAll(); | |
| 67 } | |
| 68 | |
| 69 public void deleteDocuments(Query query) throws IOException { | |
| 70 luceneWriter.deleteDocuments(query); | |
| 71 } | |
| 72 | |
| 73 public void addDocument(Map<String,Object> storedFields) throws IOException { | |
| 74 Document doc = newDocument(storedFields); | |
| 75 luceneWriter.addDocument(doc); | |
| 76 } | |
| 77 | |
| 78 public void updateDocument(String keyFieldName,Map<String,Object> storedFields) throws IOException { | |
| 79 if( !isIndexed(keyFieldName) ) | |
| 80 throw new RuntimeException("can't update using unindexed field "+keyFieldName); | |
| 81 if( fieldAnalyzer.isAdded(keyFieldName) ) | |
|
1528
3bd4d7963456
use goodjava/lucene/api
Franklin Schmidt <fschmidt@gmail.com>
parents:
1525
diff
changeset
|
82 throw new RuntimeException("can't update using analyzed field "+keyFieldName); |
| 1460 | 83 Document doc = newDocument(storedFields); |
| 84 Object keyValue = storedFields.get(keyFieldName); | |
| 85 if( keyValue==null ) | |
| 86 throw new RuntimeException("no value for field "+keyFieldName); | |
| 87 Term term = LuceneUtils.term(keyFieldName,keyValue); | |
| 88 luceneWriter.updateDocument(term,doc); | |
| 89 } | |
| 90 | |
| 91 private Document newDocument(Map<String,Object> storedFields) { | |
| 92 Document doc = new Document(); | |
|
1525
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
93 MoreFieldInfo more = goodConfig.getMoreFieldInfo(storedFields); |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
94 addFields(doc,storedFields,Field.Store.YES,more.boosts); |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
95 addFields(doc,more.unstoredFields,Field.Store.NO,more.boosts); |
| 1460 | 96 return doc; |
| 97 } | |
| 98 | |
|
1525
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
99 private void addFields( Document doc, Map<String,Object> fields, Field.Store store, Map<String,Float> boosts ) { |
| 1460 | 100 for( Map.Entry<String,Object> entry : fields.entrySet() ) { |
| 101 String name = entry.getKey(); | |
| 102 Object value = entry.getValue(); | |
|
1525
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
103 Float boost = boosts.get(name); |
| 1460 | 104 if( value instanceof List ) { |
| 105 for( Object v : (List)value ) { | |
|
1525
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
106 doc.add( newField(name,v,store,boost) ); |
| 1460 | 107 } |
| 108 } else { | |
|
1525
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
109 doc.add( newField(name,value,store,boost) ); |
| 1460 | 110 } |
| 111 } | |
| 112 } | |
| 113 | |
|
1525
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
114 private Field newField( String name, Object value, Field.Store store, Float boost ) { |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
115 Field field = newField(name,value,store); |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
116 if( boost != null ) |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
117 field.setBoost(boost); |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
118 return field; |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
119 } |
|
f848d40b3b07
lucene.api add boosts
Franklin Schmidt <fschmidt@gmail.com>
parents:
1476
diff
changeset
|
120 |
| 1460 | 121 private Field newField( String name, Object value, Field.Store store ) { |
| 122 boolean isIndexed = isIndexed(name); | |
| 123 if( store==Field.Store.NO && !isIndexed ) | |
| 124 throw new RuntimeException("field '"+name+"' is unstored and unindexed"); | |
| 125 if( value instanceof String ) { | |
| 126 String s = (String)value; | |
| 127 if( !isIndexed ) { | |
| 128 return new StoredField(name,s); | |
| 129 } else if( !fieldAnalyzer.isAdded(name) ) { | |
| 130 return new StringField(name,s,store); | |
| 131 } else { | |
| 132 return new TextField(name,s,store); | |
| 133 } | |
| 134 } else if( value instanceof Integer ) { | |
| 135 int i = (Integer)value; | |
| 136 if( !isIndexed ) { | |
| 137 return new StoredField(name,i); | |
| 138 } else { | |
| 139 return new IntField(name,i,store); | |
| 140 } | |
| 141 } else if( value instanceof Long ) { | |
| 142 long i = (Long)value; | |
| 143 if( !isIndexed ) { | |
| 144 return new StoredField(name,i); | |
| 145 } else { | |
| 146 return new LongField(name,i,store); | |
| 147 } | |
| 148 } else if( value instanceof Double ) { | |
| 149 double i = (Double)value; | |
| 150 if( !isIndexed ) { | |
| 151 return new StoredField(name,i); | |
| 152 } else { | |
| 153 return new DoubleField(name,i,store); | |
| 154 } | |
| 155 } else if( value instanceof Float ) { | |
| 156 float i = (Float)value; | |
| 157 if( !isIndexed ) { | |
| 158 return new StoredField(name,i); | |
| 159 } else { | |
| 160 return new FloatField(name,i,store); | |
| 161 } | |
| 162 } else if( value instanceof byte[] ) { | |
| 163 if( isIndexed ) | |
| 164 throw new RuntimeException("can't index byte field "+name); | |
| 165 byte[] b = (byte[])value; | |
| 166 return new StoredField(name, b); | |
| 167 } else | |
| 168 throw new RuntimeException("invalid value type "+value.getClass()+"' for field '"+name+"'"); | |
| 169 } | |
| 170 | |
| 171 private synchronized boolean isIndexed(String fieldName) { | |
| 172 Boolean b = indexedMap.get(fieldName); | |
| 173 if( b==null ) { | |
| 174 b = goodConfig.isIndexed(fieldName); | |
| 175 indexedMap.put(fieldName,b); | |
| 176 Analyzer analyzer = goodConfig.getAnalyzer(fieldName); | |
| 177 if( analyzer!=null ) | |
| 178 fieldAnalyzer.add(fieldName,analyzer); | |
| 179 } | |
| 180 return b; | |
| 181 } | |
| 182 | |
| 183 | |
| 184 public void reindexDocuments(final String keyFieldName,Query query) throws IOException { | |
| 1465 | 185 IndexReader reader = openReader(); |
| 1460 | 186 final IndexSearcher searcher = new IndexSearcher(reader); |
| 187 searcher.search( query, new GoodCollector(){ | |
| 188 public void collectDoc(int iDoc) throws IOException { | |
| 189 Document doc = searcher.doc(iDoc); | |
| 190 Map<String,Object> storedFields = LuceneUtils.toMap(doc); | |
| 191 updateDocument(keyFieldName,storedFields); | |
| 192 } | |
| 193 }); | |
| 194 reader.close(); | |
| 195 } | |
| 1465 | 196 |
| 197 public IndexReader openReader() throws IOException { | |
| 198 return DirectoryReader.open(luceneWriter.getDirectory()); | |
| 199 } | |
|
1476
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
200 |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
201 public void check() throws IOException { |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
202 CheckIndex.Status status = new CheckIndex(luceneWriter.getDirectory()).checkIndex(); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
203 if( !status.clean ) |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
204 logger.error("index not clean"); |
|
7d145095cc0b
lucene.logging check
Franklin Schmidt <fschmidt@gmail.com>
parents:
1465
diff
changeset
|
205 } |
|
1544
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
206 |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
207 public void tag(String tag) throws IOException {} |
|
35601f15ecc3
add lucene log tag and restore_from_log
Franklin Schmidt <fschmidt@gmail.com>
parents:
1539
diff
changeset
|
208 |
| 1460 | 209 } |