Mercurial Hosting > luan

comparison src/luan/modules/lucene/LuceneIndex.java @ 1562:b89212fd04b5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
remove table.luan
author Franklin Schmidt <fschmidt@gmail.com>
date Sun, 08 Nov 2020 16:50:59 -0700
parents 52241b69c339
children 8fbcc4747091
comparison
equal deleted inserted replaced
1561:e1a13e707bf3 1562:b89212fd04b5
105 synchronized(indexes) { 105 synchronized(indexes) {
106 Reference<LuceneIndex> ref = indexes.get(key); 106 Reference<LuceneIndex> ref = indexes.get(key);
107 if( ref != null ) { 107 if( ref != null ) {
108 LuceneIndex li = ref.get(); 108 LuceneIndex li = ref.get();
109 if( li != null ) { 109 if( li != null ) {
110 Object version = options.get("version"); 110 Object version = options.get(luan,"version");
111 if( version==null || version.equals(li.version) ) 111 if( version==null || version.equals(li.version) )
112 return li; 112 return li;
113 li.closeWriter(); 113 li.closeWriter();
114 } 114 }
115 } 115 }
285 wrote(); 285 wrote();
286 writeLock.unlock(); 286 writeLock.unlock();
287 } 287 }
288 } 288 }
289 289
290 public void save(LuanTable doc) 290 public void save(Luan luan,LuanTable doc)
291 throws LuanException, IOException, SQLException 291 throws LuanException, IOException, SQLException
292 { 292 {
293 Object obj = doc.get("id"); 293 Object obj = doc.get(luan,"id");
294 Long id; 294 Long id;
295 try { 295 try {
296 id = (Long)obj; 296 id = (Long)obj;
297 } catch(ClassCastException e) { 297 } catch(ClassCastException e) {
298 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName()); 298 throw new LuanException("id should be Long but is "+obj.getClass().getSimpleName());
301 boolean commit = !writeLock.isHeldByCurrentThread(); 301 boolean commit = !writeLock.isHeldByCurrentThread();
302 writeLock.lock(); 302 writeLock.lock();
303 try { 303 try {
304 if( id == null ) { 304 if( id == null ) {
305 id = ++this.id; 305 id = ++this.id;
306 doc.put("id",id); 306 doc.put(luan,"id",id);
307 if( postgresBackup != null ) 307 if( postgresBackup != null )
308 postgresBackup.add(doc); 308 postgresBackup.add(luan,doc);
309 writer.addDocument(toLucene(doc)); 309 writer.addDocument(toLucene(doc));
310 } else { 310 } else {
311 if( postgresBackup != null ) 311 if( postgresBackup != null )
312 postgresBackup.update(doc); 312 postgresBackup.update(luan,doc);
313 writer.updateDocument( "id", toLucene(doc) ); 313 writer.updateDocument( "id", toLucene(doc) );
314 } 314 }
315 if(commit) writer.commit(); 315 if(commit) writer.commit();
316 } finally { 316 } finally {
317 wrote(); 317 wrote();
411 public Object snapshot(LuanFunction fn) throws LuanException, IOException { 411 public Object snapshot(LuanFunction fn) throws LuanException, IOException {
412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy(); 412 SnapshotDeletionPolicy snapshotDeletionPolicy = snapshotDeletionPolicy();
413 IndexCommit ic = snapshotDeletionPolicy.snapshot(); 413 IndexCommit ic = snapshotDeletionPolicy.snapshot();
414 try { 414 try {
415 String dir = fsDir.getDirectory().toString(); 415 String dir = fsDir.getDirectory().toString();
416 LuanTable fileNames = new LuanTable(fn.luan(),new ArrayList(ic.getFileNames())); 416 LuanTable fileNames = new LuanTable(new ArrayList(ic.getFileNames()));
417 return fn.call(dir,fileNames); 417 return fn.call(dir,fileNames);
418 } finally { 418 } finally {
419 snapshotDeletionPolicy.release(ic); 419 snapshotDeletionPolicy.release(ic);
420 } 420 }
421 } 421 }
470 this.query = query; 470 this.query = query;
471 } 471 }
472 472
473 @Override public Object call(Object[] args) throws LuanException { 473 @Override public Object call(Object[] args) throws LuanException {
474 try { 474 try {
475 LuanTable doc = toTable(luan(),searcher.doc(docID)); 475 LuanTable doc = toTable(searcher.doc(docID));
476 if( args.length > 0 && "explain".equals(args[0]) ) { 476 if( args.length > 0 && "explain".equals(args[0]) ) {
477 Explanation explanation = searcher.explain(query,docID); 477 Explanation explanation = searcher.explain(query,docID);
478 return new Object[]{doc,explanation}; 478 return new Object[]{doc,explanation};
479 } else { 479 } else {
480 return doc; 480 return doc;
603 603
604 static Map<String,Object> toLucene(LuanTable table) throws LuanException { 604 static Map<String,Object> toLucene(LuanTable table) throws LuanException {
605 return SupplementingConfig.toLucene(table); 605 return SupplementingConfig.toLucene(table);
606 } 606 }
607 607
608 private static LuanTable toTable(Luan luan,Document doc) throws LuanException { 608 private static LuanTable toTable(Document doc) throws LuanException {
609 return doc==null ? null : SupplementingConfig.toTable(luan,LuceneUtils.toMap(doc)); 609 return doc==null ? null : SupplementingConfig.toTable(LuceneUtils.toMap(doc));
610 } 610 }
611 611
612 612
613 private static final Formatter nullFormatter = new Formatter() { 613 private static final Formatter nullFormatter = new Formatter() {
614 public String highlightTerm(String originalText,TokenGroup tokenGroup) { 614 public String highlightTerm(String originalText,TokenGroup tokenGroup) {
704 Query query = new PrefixQuery(new Term("id")); 704 Query query = new PrefixQuery(new Term("id"));
705 MyCollector col = new MyCollector() { 705 MyCollector col = new MyCollector() {
706 @Override public void collect(int iDoc) throws IOException { 706 @Override public void collect(int iDoc) throws IOException {
707 try { 707 try {
708 Document doc = searcher.doc( docBase + iDoc ); 708 Document doc = searcher.doc( docBase + iDoc );
709 LuanTable tbl = toTable(luan,doc); 709 LuanTable tbl = toTable(doc);
710 postgresBackup.add(tbl); 710 postgresBackup.add(luan,tbl);
711 } catch(LuanException e) { 711 } catch(LuanException e) {
712 throw new LuanRuntimeException(e); 712 throw new LuanRuntimeException(e);
713 } catch(SQLException e) { 713 } catch(SQLException e) {
714 throw new RuntimeException(e); 714 throw new RuntimeException(e);
715 } 715 }
870 final List<Long> idsPostgres = postgresChecker.getIds(); 870 final List<Long> idsPostgres = postgresChecker.getIds();
871 final int nLucene = idsLucene.size(); 871 final int nLucene = idsLucene.size();
872 final int nPostgres = idsPostgres.size(); 872 final int nPostgres = idsPostgres.size();
873 int iLucene = 0; 873 int iLucene = 0;
874 int iPostgres = 0; 874 int iPostgres = 0;
875 LuanToString lts = new LuanToString(null,null); 875 LuanToString lts = new LuanToString(luan,null,null);
876 lts.settingsInit.strict = true; 876 lts.settingsInit.strict = true;
877 lts.settingsInit.numberTypes = true; 877 lts.settingsInit.numberTypes = true;
878 while( iLucene < nLucene && iPostgres < nPostgres ) { 878 while( iLucene < nLucene && iPostgres < nPostgres ) {
879 long idLucene = idsLucene.get(iLucene); 879 long idLucene = idsLucene.get(iLucene);
880 long idPostgres = idsPostgres.get(iPostgres); 880 long idPostgres = idsPostgres.get(iPostgres);
881 if( idLucene < idPostgres ) { 881 if( idLucene < idPostgres ) {
882 iLucene++; 882 iLucene++;
883 checkPostgres(luan,postgresChecker,lts,idLucene); 883 checkPostgres(postgresChecker,lts,idLucene);
884 } else if( idLucene > idPostgres ) { 884 } else if( idLucene > idPostgres ) {
885 iPostgres++; 885 iPostgres++;
886 checkPostgres(luan,postgresChecker,lts,idPostgres); 886 checkPostgres(postgresChecker,lts,idPostgres);
887 } else { // == 887 } else { // ==
888 LuanTable docPostgres = postgresChecker.getDoc(idPostgres); 888 LuanTable docPostgres = postgresChecker.getDoc(idPostgres);
889 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1); 889 TopDocs td = searcher.search(new TermQuery(term("id",idLucene)),1);
890 if( td.totalHits != 1 ) throw new RuntimeException(); 890 if( td.totalHits != 1 ) throw new RuntimeException();
891 Document doc = searcher.doc( td.scoreDocs[0].doc ); 891 Document doc = searcher.doc( td.scoreDocs[0].doc );
892 LuanTable docLucene = toTable(luan,doc); 892 LuanTable docLucene = toTable(doc);
893 if( !equal(docPostgres,docLucene) ) { 893 if( !equal(docPostgres,docLucene) ) {
894 checkPostgres(luan,postgresChecker,lts,idPostgres); 894 checkPostgres(postgresChecker,lts,idPostgres);
895 } 895 }
896 iLucene++; 896 iLucene++;
897 iPostgres++; 897 iPostgres++;
898 } 898 }
899 } 899 }
900 while( iLucene < nLucene ) { 900 while( iLucene < nLucene ) {
901 long idLucene = idsLucene.get(iLucene++); 901 long idLucene = idsLucene.get(iLucene++);
902 checkPostgres(luan,postgresChecker,lts,idLucene); 902 checkPostgres(postgresChecker,lts,idLucene);
903 } 903 }
904 while( iPostgres < nPostgres ) { 904 while( iPostgres < nPostgres ) {
905 long idPostgres = idsPostgres.get(iPostgres++); 905 long idPostgres = idsPostgres.get(iPostgres++);
906 checkPostgres(luan,postgresChecker,lts,idPostgres); 906 checkPostgres(postgresChecker,lts,idPostgres);
907 } 907 }
908 } finally { 908 } finally {
909 close(searcher); 909 close(searcher);
910 postgresChecker.close(); 910 postgresChecker.close();
911 } 911 }
912 } 912 }
913 913
914 private void checkPostgres(Luan luan,PostgresBackup.Checker postgresChecker,LuanToString lts,long id) 914 private void checkPostgres(PostgresBackup.Checker postgresChecker,LuanToString lts,long id)
915 throws IOException, SQLException, LuanException, ParseException 915 throws IOException, SQLException, LuanException, ParseException
916 { 916 {
917 //logger.info("check id "+id); 917 //logger.info("check id "+id);
918 writeLock.lock(); 918 writeLock.lock();
919 try { 919 try {
924 LuanTable docLucene; 924 LuanTable docLucene;
925 if( td.totalHits == 0 ) { 925 if( td.totalHits == 0 ) {
926 docLucene = null; 926 docLucene = null;
927 } else if( td.totalHits == 1 ) { 927 } else if( td.totalHits == 1 ) {
928 Document doc = searcher.doc( td.scoreDocs[0].doc ); 928 Document doc = searcher.doc( td.scoreDocs[0].doc );
929 docLucene = toTable(luan,doc); 929 docLucene = toTable(doc);
930 } else 930 } else
931 throw new RuntimeException(); 931 throw new RuntimeException();
932 if( docPostgres == null ) { 932 if( docPostgres == null ) {
933 if( docLucene != null ) 933 if( docLucene != null )
934 logger.error("id "+id+" found in lucene but not postgres"); 934 logger.error("id "+id+" found in lucene but not postgres");