Mercurial Hosting > luan
changeset 1344:dc2af9d5463b
move queryparser to lib
| author | Franklin Schmidt <fschmidt@gmail.com> |
|---|---|
| date | Mon, 25 Feb 2019 07:00:55 -0700 |
| parents | 7d9a1f8894b0 |
| children | 6f8988830098 |
| files | src/luan/lib/queryparser/FieldParser.java src/luan/lib/queryparser/MultiFieldParser.java src/luan/lib/queryparser/NumberFieldParser.java src/luan/lib/queryparser/SaneQueryParser.java src/luan/lib/queryparser/StringFieldParser.java src/luan/lib/queryparser/SynonymParser.java src/luan/modules/lucene/Lucene.luan src/luan/modules/lucene/LuceneIndex.java src/luan/modules/lucene/queryparser/FieldParser.java src/luan/modules/lucene/queryparser/MultiFieldParser.java src/luan/modules/lucene/queryparser/NumberFieldParser.java src/luan/modules/lucene/queryparser/SaneQueryParser.java src/luan/modules/lucene/queryparser/StringFieldParser.java src/luan/modules/lucene/queryparser/SynonymParser.java |
| diffstat | 14 files changed, 606 insertions(+), 606 deletions(-) [+] |
line wrap: on
line diff
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/lib/queryparser/FieldParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/lib/queryparser/FieldParser.java Mon Feb 25 07:00:55 2019 -0700 @@ -0,0 +1,12 @@ +package luan.lib.queryparser; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import luan.lib.parser.ParseException; + + +public interface FieldParser { + public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException; + public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException; + public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) throws ParseException; +}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/lib/queryparser/MultiFieldParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/lib/queryparser/MultiFieldParser.java Mon Feb 25 07:00:55 2019 -0700 @@ -0,0 +1,86 @@ +package luan.lib.queryparser; + +import java.util.Map; +import java.util.HashMap; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.SortField; +import luan.lib.parser.ParseException; + + +public class MultiFieldParser implements FieldParser { + + /** + * maps field name to FieldParser + */ + public final Map<String,FieldParser> fields = new HashMap<String,FieldParser>(); + public boolean allowUnspecifiedFields = false; + private final FieldParser defaultFieldParser; + private final String[] defaultFields; + + public MultiFieldParser() { + this.defaultFieldParser = null; + this.defaultFields = null; + } + + public MultiFieldParser(FieldParser defaultFieldParser,String... defaultFields) { + this.defaultFieldParser = defaultFieldParser; + this.defaultFields = defaultFields; + for( String field : defaultFields ) { + fields.put(field,defaultFieldParser); + } + } + + @Override public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { + if( field == null ) { + if( defaultFieldParser == null ) + throw qp.exception("no defaults were specified, so a field is required"); + if( defaultFields.length == 1 ) + return defaultFieldParser.getQuery(qp,defaultFields[0],query); + BooleanQuery bq = new BooleanQuery(); + for( String f : defaultFields ) { + bq.add( defaultFieldParser.getQuery(qp,f,query), BooleanClause.Occur.SHOULD ); + } + return bq; + } else { + FieldParser fp = fields.get(field); + if( fp != null ) + return fp.getQuery(qp,field,query); + if( allowUnspecifiedFields ) + return defaultFieldParser.getQuery(qp,field,query); + throw qp.exception("unrecognized field '"+field+"'"); + } + } + + @Override public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { + if( field == null ) { + if( defaultFieldParser == null ) + throw qp.exception("no defaults were specified, so a field is required"); + if( defaultFields.length == 1 ) + return defaultFieldParser.getRangeQuery(qp,defaultFields[0],minQuery,maxQuery,includeMin,includeMax); + BooleanQuery bq = new BooleanQuery(); + for( String f : defaultFields ) { + bq.add( defaultFieldParser.getRangeQuery(qp,f,minQuery,maxQuery,includeMin,includeMax), BooleanClause.Occur.SHOULD ); + } + return bq; + } else { + FieldParser fp = fields.get(field); + if( fp != null ) + return fp.getRangeQuery(qp,field,minQuery,maxQuery,includeMin,includeMax); + if( allowUnspecifiedFields ) + return defaultFieldParser.getRangeQuery(qp,field,minQuery,maxQuery,includeMin,includeMax); + throw qp.exception("field '"+field+"' not specified"); + } + } + + @Override public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) throws ParseException { + FieldParser fp = fields.get(field); + if( fp != null ) + return fp.getSortField(qp,field,reverse); + if( allowUnspecifiedFields ) + return defaultFieldParser.getSortField(qp,field,reverse); + throw qp.exception("field '"+field+"' not specified"); + } + +}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/lib/queryparser/NumberFieldParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/lib/queryparser/NumberFieldParser.java Mon Feb 25 07:00:55 2019 -0700 @@ -0,0 +1,84 @@ +package luan.lib.queryparser; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.SortField; +import luan.lib.parser.ParseException; + + +public abstract class NumberFieldParser implements FieldParser { + + @Override public final Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { + return getRangeQuery(qp,field,query,query,true,true); + } + + @Override public final Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { + try { + return getRangeQuery(field,minQuery,maxQuery,includeMin,includeMax); + } catch(NumberFormatException e) { + throw qp.exception(e); + } + } + + abstract protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax); + + @Override public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) { + return new SortField( field, sortType(), reverse ); + } + + abstract protected SortField.Type sortType(); + + + public static final FieldParser INT = new NumberFieldParser() { + + @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { + int min = Integer.parseInt(minQuery); + int max = Integer.parseInt(maxQuery); + return NumericRangeQuery.newIntRange(field,min,max,includeMin,includeMax); + } + + @Override protected SortField.Type sortType() { + return SortField.Type.INT; + } + }; + + public static final FieldParser LONG = new NumberFieldParser() { + + @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { + long min = Long.parseLong(minQuery); + long max = Long.parseLong(maxQuery); + return NumericRangeQuery.newLongRange(field,min,max,includeMin,includeMax); + } + + @Override protected SortField.Type sortType() { + return SortField.Type.LONG; + } + }; + + public static final FieldParser FLOAT = new NumberFieldParser() { + + @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { + float min = Float.parseFloat(minQuery); + float max = Float.parseFloat(maxQuery); + return NumericRangeQuery.newFloatRange(field,min,max,includeMin,includeMax); + } + + @Override protected SortField.Type sortType() { + return SortField.Type.FLOAT; + } + }; + + public static final FieldParser DOUBLE = new NumberFieldParser() { + + @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { + double min = Double.parseDouble(minQuery); + double max = Double.parseDouble(maxQuery); + return NumericRangeQuery.newDoubleRange(field,min,max,includeMin,includeMax); + } + + @Override protected SortField.Type sortType() { + return SortField.Type.DOUBLE; + } + }; + +}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/lib/queryparser/SaneQueryParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/lib/queryparser/SaneQueryParser.java Mon Feb 25 07:00:55 2019 -0700 @@ -0,0 +1,260 @@ +package luan.lib.queryparser; + +import java.util.List; +import java.util.ArrayList; +import java.util.regex.Pattern; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import luan.lib.parser.Parser; +import luan.lib.parser.ParseException; + + +public class SaneQueryParser { + + public static Query parseQuery(FieldParser fieldParser,String query) throws ParseException { + return new SaneQueryParser(fieldParser,query).parseQuery(); + } + + private static Pattern specialChar = Pattern.compile("[ \\t\\r\\n\":\\[\\]{}^+\\-(),?*\\\\]"); + + public static String literal(String s) { + return specialChar.matcher(s).replaceAll("\\\\$0"); + } + + public static Sort parseSort(FieldParser fieldParser,String sort) throws ParseException { + return new SaneQueryParser(fieldParser,sort).parseSort(); + } + + + private static final String NOT_IN_TERM = " \t\r\n\":[]{}^+-()"; + private static final String NOT_IN_FIELD = NOT_IN_TERM + ","; + private final FieldParser fieldParser; + private final Parser parser; + + private SaneQueryParser(FieldParser fieldParser,String query) { + this.fieldParser = fieldParser; + this.parser = new Parser(query); + parser.begin(); + } + + ParseException exception(String msg) { + parser.failure(); + return new ParseException(parser,msg); + } + + ParseException exception(Exception cause) { + parser.failure(); + return new ParseException(parser,cause); + } + + private Query parseQuery() throws ParseException { + Spaces(); + BooleanQuery bq = new BooleanQuery(); + while( !parser.endOfInput() ) { + bq.add( Term(null) ); + } + BooleanClause[] clauses = bq.getClauses(); + switch( clauses.length ) { + case 0: + return new MatchAllDocsQuery(); + case 1: + { + BooleanClause bc = clauses[0]; + if( bc.getOccur() != BooleanClause.Occur.MUST_NOT ) + return bc.getQuery(); + } + default: + return bq; + } + } + + private BooleanClause Term(String defaultField) throws ParseException { + BooleanClause.Occur occur; + if( parser.match('+') ) { + occur = BooleanClause.Occur.MUST; + Spaces(); + } else if( parser.match('-') ) { + occur = BooleanClause.Occur.MUST_NOT; + Spaces(); + } else { + occur = BooleanClause.Occur.SHOULD; + } + String field = QueryField(); + if( field == null ) + field = defaultField; + Query query = NestedTerm(field); + if( query == null ) + query = RangeTerm(field); + if( query == null ) { + parser.begin(); + String match = SimpleTerm(); + query = fieldParser.getQuery(this,field,match); + parser.success(); + } + if( parser.match('^') ) { + Spaces(); + int start = parser.begin(); + try { + while( parser.anyOf("0123456789.") ); + String match = parser.textFrom(start); + float boost = Float.parseFloat(match); + query.setBoost(boost); + } catch(NumberFormatException e) { + throw exception(e); + } + parser.success(); + Spaces(); + } + BooleanClause bc = new BooleanClause(query,occur); + return bc; + } + + private Query NestedTerm(String field) throws ParseException { + parser.begin(); + if( !parser.match('(') ) + return parser.failure(null); + BooleanQuery bq = new BooleanQuery(); + while( !parser.match(')') ) { + if( parser.endOfInput() ) + throw exception("unclosed parentheses"); + bq.add( Term(field) ); + } + Spaces(); + BooleanClause[] clauses = bq.getClauses(); + switch( clauses.length ) { + case 0: + throw exception("empty parentheses"); + case 1: + { + BooleanClause bc = clauses[0]; + if( bc.getOccur() != BooleanClause.Occur.MUST_NOT ) + return parser.success(bc.getQuery()); + } + default: + return parser.success(bq); + } + } + + private Query RangeTerm(String field) throws ParseException { + parser.begin(); + if( !parser.anyOf("[{") ) + return parser.failure(null); + boolean includeMin = parser.lastChar() == '['; + Spaces(); + String minQuery = SimpleTerm(); + TO(); + String maxQuery = SimpleTerm(); + if( !parser.anyOf("]}") ) + throw exception("unclosed range"); + boolean includeMax = parser.lastChar() == ']'; + Spaces(); + Query query = fieldParser.getRangeQuery(this,field,minQuery,maxQuery,includeMin,includeMax); + return parser.success(query); + } + + private void TO() throws ParseException { + parser.begin(); + if( !(parser.match("TO") && Space()) ) + throw exception("'TO' expected"); + Spaces(); + parser.success(); + } + + private String SimpleTerm() throws ParseException { + parser.begin(); + String match; + if( parser.match('"') ) { + int start = parser.currentIndex() - 1; + while( !parser.match('"') ) { + if( parser.endOfInput() ) + throw exception("unclosed quotes"); + parser.anyChar(); + checkEscape(); + } + match = parser.textFrom(start); + Spaces(); + } else { + match = Unquoted(NOT_IN_TERM); + } + if( match.length() == 0 ) + throw exception("invalid input"); + return parser.success(match); + } + + private String QueryField() throws ParseException { + parser.begin(); + String match = Field(); + if( match==null || !parser.match(':') ) + return parser.failure((String)null); + Spaces(); + return parser.success(match); + } + + private String Field() throws ParseException { + parser.begin(); + String match = Unquoted(NOT_IN_FIELD); + if( match.length()==0 ) + return parser.failure((String)null); + match = StringFieldParser.escape(this,match); + return parser.success(match); + } + + private String Unquoted(String exclude) throws ParseException { + int start = parser.begin(); + while( parser.noneOf(exclude) ) { + checkEscape(); + } + String match = parser.textFrom(start); + Spaces(); + return parser.success(match); + } + + private void checkEscape() { + if( parser.lastChar() == '\\' ) + parser.anyChar(); + } + + private void Spaces() { + while( Space() ); + } + + private boolean Space() { + return parser.anyOf(" \t\r\n"); + } + + + // sort + + private Sort parseSort() throws ParseException { + Spaces(); + if( parser.endOfInput() ) + return null; + List<SortField> list = new ArrayList<SortField>(); + list.add( SortField() ); + while( !parser.endOfInput() ) { + parser.begin(); + if( !parser.match(',') ) + throw exception("',' expected"); + Spaces(); + parser.success(); + list.add( SortField() ); + } + return new Sort(list.toArray(new SortField[0])); + } + + private SortField SortField() throws ParseException { + parser.begin(); + String field = Field(); + if( field==null ) + throw exception("invalid input"); + boolean reverse = !parser.matchIgnoreCase("asc") && parser.matchIgnoreCase("desc"); + Spaces(); + SortField sf = fieldParser.getSortField(this,field,reverse); + return parser.success(sf); + } + +}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/lib/queryparser/StringFieldParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/lib/queryparser/StringFieldParser.java Mon Feb 25 07:00:55 2019 -0700 @@ -0,0 +1,113 @@ +package luan.lib.queryparser; + +import java.io.StringReader; +import java.io.IOException; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.SortField; +import org.apache.lucene.index.Term; +import luan.lib.parser.ParseException; + + +public class StringFieldParser implements FieldParser { + public int slop = 0; + public final Analyzer analyzer; + + public StringFieldParser(Analyzer analyzer) { + this.analyzer = analyzer; + } + + @Override public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { + String wildcard = wildcard(qp,query); + if( wildcard != null ) + return new WildcardQuery(new Term(field,wildcard)); + if( query.endsWith("*") && !query.endsWith("\\*") ) + return new PrefixQuery(new Term(field,query.substring(0,query.length()-1))); + query = escape(qp,query); + PhraseQuery pq = new PhraseQuery(); + try { + TokenStream ts = analyzer.tokenStream(field,new StringReader(query)); + CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class); + PositionIncrementAttribute posAttr = ts.addAttribute(PositionIncrementAttribute.class); + ts.reset(); + int pos = -1; + while( ts.incrementToken() ) { + pos += posAttr.getPositionIncrement(); + pq.add( new Term(field,termAttr.toString()), pos ); + } + ts.end(); + ts.close(); + } catch(IOException e) { + throw new RuntimeException(e); + } + Term[] terms = pq.getTerms(); + if( terms.length==1 && pq.getPositions()[0]==0 ) + return new TermQuery(terms[0]); + return pq; + } + + @Override public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { + minQuery = escape(qp,minQuery); + maxQuery = escape(qp,maxQuery); + return TermRangeQuery.newStringRange(field,minQuery,maxQuery,includeMin,includeMax); + } + + static String escape(SaneQueryParser qp,String s) throws ParseException { + final char[] a = s.toCharArray(); + int i, n; + if( a[0] == '"' ) { + if( a[a.length-1] != '"' ) throw new RuntimeException(); + i = 1; + n = a.length - 1; + } else { + i = 0; + n = a.length; + } + StringBuilder sb = new StringBuilder(); + for( ; i<n; i++ ) { + char c = a[i]; + if( c == '\\' ) { + if( ++i == a.length ) + throw qp.exception("ends with '\\'"); + c = a[i]; + } + sb.append(c); + } + return sb.toString(); + } + + private static String wildcard(SaneQueryParser qp,String s) throws ParseException { + final char[] a = s.toCharArray(); + if( a[0] == '"' ) + return null; + boolean hasWildcard = false; + StringBuilder sb = new StringBuilder(); + for( int i=0; i<a.length; i++ ) { + char c = a[i]; + if( c=='?' || c=='*' && i<a.length-1 ) + hasWildcard = true; + if( c == '\\' ) { + if( ++i == a.length ) + throw qp.exception("ends with '\\'"); + c = a[i]; + if( c=='?' || c=='*' ) + sb.append('\\'); + } + sb.append(c); + } + return hasWildcard ? sb.toString() : null; + } + + @Override public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) { + return new SortField( field, SortField.Type.STRING, reverse ); + } + +}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/lib/queryparser/SynonymParser.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/luan/lib/queryparser/SynonymParser.java Mon Feb 25 07:00:55 2019 -0700 @@ -0,0 +1,43 @@ +package luan.lib.queryparser; + +import java.util.Map; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.SortField; +import luan.lib.parser.ParseException; + + +public class SynonymParser implements FieldParser { + private final FieldParser fp; + private final Map<String,String[]> synonymMap; + + public SynonymParser(FieldParser fp,Map<String,String[]> synonymMap) { + this.fp = fp; + this.synonymMap = synonymMap; + } + + protected String[] getSynonyms(String query) { + return synonymMap.get(query); + } + + public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { + String[] synonyms = getSynonyms(query); + if( synonyms == null ) + return fp.getQuery(qp,field,query); + BooleanQuery bq = new BooleanQuery(); + bq.add( fp.getQuery(qp,field,query), BooleanClause.Occur.SHOULD ); + for( String s : synonyms ) { + bq.add( fp.getQuery(qp,field,s), BooleanClause.Occur.SHOULD ); + } + return bq; + } + + public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { + return fp.getRangeQuery(qp,field,minQuery,maxQuery,includeMin,includeMax); + } + + public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) throws ParseException { + return fp.getSortField(qp,field,reverse); + } +}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/Lucene.luan --- a/src/luan/modules/lucene/Lucene.luan Fri Feb 22 10:12:05 2019 -0700 +++ b/src/luan/modules/lucene/Lucene.luan Mon Feb 25 07:00:55 2019 -0700 @@ -11,9 +11,9 @@ local matches = String.matches or error() local Rpc = require "luan:Rpc.luan" local LuceneIndex = require "java:luan.modules.lucene.LuceneIndex" -local NumberFieldParser = require "java:luan.modules.lucene.queryparser.NumberFieldParser" -local StringFieldParser = require "java:luan.modules.lucene.queryparser.StringFieldParser" -local SaneQueryParser = require "java:luan.modules.lucene.queryparser.SaneQueryParser" +local NumberFieldParser = require "java:luan.lib.queryparser.NumberFieldParser" +local StringFieldParser = require "java:luan.lib.queryparser.StringFieldParser" +local SaneQueryParser = require "java:luan.lib.queryparser.SaneQueryParser" local Version = require "java:org.apache.lucene.util.Version" local EnglishAnalyzer = require "java:org.apache.lucene.analysis.en.EnglishAnalyzer"
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/LuceneIndex.java --- a/src/luan/modules/lucene/LuceneIndex.java Fri Feb 22 10:12:05 2019 -0700 +++ b/src/luan/modules/lucene/LuceneIndex.java Mon Feb 25 07:00:55 2019 -0700 @@ -62,11 +62,11 @@ import org.apache.lucene.search.highlight.SimpleSpanFragmenter; import org.apache.lucene.search.highlight.QueryScorer; import org.apache.lucene.search.highlight.TokenGroup; -import luan.modules.lucene.queryparser.SaneQueryParser; -import luan.modules.lucene.queryparser.FieldParser; -import luan.modules.lucene.queryparser.MultiFieldParser; -import luan.modules.lucene.queryparser.StringFieldParser; -import luan.modules.lucene.queryparser.NumberFieldParser; +import luan.lib.queryparser.SaneQueryParser; +import luan.lib.queryparser.FieldParser; +import luan.lib.queryparser.MultiFieldParser; +import luan.lib.queryparser.StringFieldParser; +import luan.lib.queryparser.NumberFieldParser; import luan.lib.parser.ParseException; import luan.modules.Utils; import luan.Luan;
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/queryparser/FieldParser.java --- a/src/luan/modules/lucene/queryparser/FieldParser.java Fri Feb 22 10:12:05 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,12 +0,0 @@ -package luan.modules.lucene.queryparser; - -import org.apache.lucene.search.Query; -import org.apache.lucene.search.SortField; -import luan.lib.parser.ParseException; - - -public interface FieldParser { - public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException; - public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException; - public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) throws ParseException; -}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/queryparser/MultiFieldParser.java --- a/src/luan/modules/lucene/queryparser/MultiFieldParser.java Fri Feb 22 10:12:05 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,86 +0,0 @@ -package luan.modules.lucene.queryparser; - -import java.util.Map; -import java.util.HashMap; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.SortField; -import luan.lib.parser.ParseException; - - -public class MultiFieldParser implements FieldParser { - - /** - * maps field name to FieldParser - */ - public final Map<String,FieldParser> fields = new HashMap<String,FieldParser>(); - public boolean allowUnspecifiedFields = false; - private final FieldParser defaultFieldParser; - private final String[] defaultFields; - - public MultiFieldParser() { - this.defaultFieldParser = null; - this.defaultFields = null; - } - - public MultiFieldParser(FieldParser defaultFieldParser,String... defaultFields) { - this.defaultFieldParser = defaultFieldParser; - this.defaultFields = defaultFields; - for( String field : defaultFields ) { - fields.put(field,defaultFieldParser); - } - } - - @Override public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { - if( field == null ) { - if( defaultFieldParser == null ) - throw qp.exception("no defaults were specified, so a field is required"); - if( defaultFields.length == 1 ) - return defaultFieldParser.getQuery(qp,defaultFields[0],query); - BooleanQuery bq = new BooleanQuery(); - for( String f : defaultFields ) { - bq.add( defaultFieldParser.getQuery(qp,f,query), BooleanClause.Occur.SHOULD ); - } - return bq; - } else { - FieldParser fp = fields.get(field); - if( fp != null ) - return fp.getQuery(qp,field,query); - if( allowUnspecifiedFields ) - return defaultFieldParser.getQuery(qp,field,query); - throw qp.exception("unrecognized field '"+field+"'"); - } - } - - @Override public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { - if( field == null ) { - if( defaultFieldParser == null ) - throw qp.exception("no defaults were specified, so a field is required"); - if( defaultFields.length == 1 ) - return defaultFieldParser.getRangeQuery(qp,defaultFields[0],minQuery,maxQuery,includeMin,includeMax); - BooleanQuery bq = new BooleanQuery(); - for( String f : defaultFields ) { - bq.add( defaultFieldParser.getRangeQuery(qp,f,minQuery,maxQuery,includeMin,includeMax), BooleanClause.Occur.SHOULD ); - } - return bq; - } else { - FieldParser fp = fields.get(field); - if( fp != null ) - return fp.getRangeQuery(qp,field,minQuery,maxQuery,includeMin,includeMax); - if( allowUnspecifiedFields ) - return defaultFieldParser.getRangeQuery(qp,field,minQuery,maxQuery,includeMin,includeMax); - throw qp.exception("field '"+field+"' not specified"); - } - } - - @Override public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) throws ParseException { - FieldParser fp = fields.get(field); - if( fp != null ) - return fp.getSortField(qp,field,reverse); - if( allowUnspecifiedFields ) - return defaultFieldParser.getSortField(qp,field,reverse); - throw qp.exception("field '"+field+"' not specified"); - } - -}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/queryparser/NumberFieldParser.java --- a/src/luan/modules/lucene/queryparser/NumberFieldParser.java Fri Feb 22 10:12:05 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,84 +0,0 @@ -package luan.modules.lucene.queryparser; - -import org.apache.lucene.search.Query; -import org.apache.lucene.search.NumericRangeQuery; -import org.apache.lucene.search.SortField; -import luan.lib.parser.ParseException; - - -public abstract class NumberFieldParser implements FieldParser { - - @Override public final Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { - return getRangeQuery(qp,field,query,query,true,true); - } - - @Override public final Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { - try { - return getRangeQuery(field,minQuery,maxQuery,includeMin,includeMax); - } catch(NumberFormatException e) { - throw qp.exception(e); - } - } - - abstract protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax); - - @Override public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) { - return new SortField( field, sortType(), reverse ); - } - - abstract protected SortField.Type sortType(); - - - public static final FieldParser INT = new NumberFieldParser() { - - @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { - int min = Integer.parseInt(minQuery); - int max = Integer.parseInt(maxQuery); - return NumericRangeQuery.newIntRange(field,min,max,includeMin,includeMax); - } - - @Override protected SortField.Type sortType() { - return SortField.Type.INT; - } - }; - - public static final FieldParser LONG = new NumberFieldParser() { - - @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { - long min = Long.parseLong(minQuery); - long max = Long.parseLong(maxQuery); - return NumericRangeQuery.newLongRange(field,min,max,includeMin,includeMax); - } - - @Override protected SortField.Type sortType() { - return SortField.Type.LONG; - } - }; - - public static final FieldParser FLOAT = new NumberFieldParser() { - - @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { - float min = Float.parseFloat(minQuery); - float max = Float.parseFloat(maxQuery); - return NumericRangeQuery.newFloatRange(field,min,max,includeMin,includeMax); - } - - @Override protected SortField.Type sortType() { - return SortField.Type.FLOAT; - } - }; - - public static final FieldParser DOUBLE = new NumberFieldParser() { - - @Override protected Query getRangeQuery(String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) { - double min = Double.parseDouble(minQuery); - double max = Double.parseDouble(maxQuery); - return NumericRangeQuery.newDoubleRange(field,min,max,includeMin,includeMax); - } - - @Override protected SortField.Type sortType() { - return SortField.Type.DOUBLE; - } - }; - -}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/queryparser/SaneQueryParser.java --- a/src/luan/modules/lucene/queryparser/SaneQueryParser.java Fri Feb 22 10:12:05 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,260 +0,0 @@ -package luan.modules.lucene.queryparser; - -import java.util.List; -import java.util.ArrayList; -import java.util.regex.Pattern; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import luan.lib.parser.Parser; -import luan.lib.parser.ParseException; - - -public class SaneQueryParser { - - public static Query parseQuery(FieldParser fieldParser,String query) throws ParseException { - return new SaneQueryParser(fieldParser,query).parseQuery(); - } - - private static Pattern specialChar = Pattern.compile("[ \\t\\r\\n\":\\[\\]{}^+\\-(),?*\\\\]"); - - public static String literal(String s) { - return specialChar.matcher(s).replaceAll("\\\\$0"); - } - - public static Sort parseSort(FieldParser fieldParser,String sort) throws ParseException { - return new SaneQueryParser(fieldParser,sort).parseSort(); - } - - - private static final String NOT_IN_TERM = " \t\r\n\":[]{}^+-()"; - private static final String NOT_IN_FIELD = NOT_IN_TERM + ","; - private final FieldParser fieldParser; - private final Parser parser; - - private SaneQueryParser(FieldParser fieldParser,String query) { - this.fieldParser = fieldParser; - this.parser = new Parser(query); - parser.begin(); - } - - ParseException exception(String msg) { - parser.failure(); - return new ParseException(parser,msg); - } - - ParseException exception(Exception cause) { - parser.failure(); - return new ParseException(parser,cause); - } - - private Query parseQuery() throws ParseException { - Spaces(); - BooleanQuery bq = new BooleanQuery(); - while( !parser.endOfInput() ) { - bq.add( Term(null) ); - } - BooleanClause[] clauses = bq.getClauses(); - switch( clauses.length ) { - case 0: - return new MatchAllDocsQuery(); - case 1: - { - BooleanClause bc = clauses[0]; - if( bc.getOccur() != BooleanClause.Occur.MUST_NOT ) - return bc.getQuery(); - } - default: - return bq; - } - } - - private BooleanClause Term(String defaultField) throws ParseException { - BooleanClause.Occur occur; - if( parser.match('+') ) { - occur = BooleanClause.Occur.MUST; - Spaces(); - } else if( parser.match('-') ) { - occur = BooleanClause.Occur.MUST_NOT; - Spaces(); - } else { - occur = BooleanClause.Occur.SHOULD; - } - String field = QueryField(); - if( field == null ) - field = defaultField; - Query query = NestedTerm(field); - if( query == null ) - query = RangeTerm(field); - if( query == null ) { - parser.begin(); - String match = SimpleTerm(); - query = fieldParser.getQuery(this,field,match); - parser.success(); - } - if( parser.match('^') ) { - Spaces(); - int start = parser.begin(); - try { - while( parser.anyOf("0123456789.") ); - String match = parser.textFrom(start); - float boost = Float.parseFloat(match); - query.setBoost(boost); - } catch(NumberFormatException e) { - throw exception(e); - } - parser.success(); - Spaces(); - } - BooleanClause bc = new BooleanClause(query,occur); - return bc; - } - - private Query NestedTerm(String field) throws ParseException { - parser.begin(); - if( !parser.match('(') ) - return parser.failure(null); - BooleanQuery bq = new BooleanQuery(); - while( !parser.match(')') ) { - if( parser.endOfInput() ) - throw exception("unclosed parentheses"); - bq.add( Term(field) ); - } - Spaces(); - BooleanClause[] clauses = bq.getClauses(); - switch( clauses.length ) { - case 0: - throw exception("empty parentheses"); - case 1: - { - BooleanClause bc = clauses[0]; - if( bc.getOccur() != BooleanClause.Occur.MUST_NOT ) - return parser.success(bc.getQuery()); - } - default: - return parser.success(bq); - } - } - - private Query RangeTerm(String field) throws ParseException { - parser.begin(); - if( !parser.anyOf("[{") ) - return parser.failure(null); - boolean includeMin = parser.lastChar() == '['; - Spaces(); - String minQuery = SimpleTerm(); - TO(); - String maxQuery = SimpleTerm(); - if( !parser.anyOf("]}") ) - throw exception("unclosed range"); - boolean includeMax = parser.lastChar() == ']'; - Spaces(); - Query query = fieldParser.getRangeQuery(this,field,minQuery,maxQuery,includeMin,includeMax); - return parser.success(query); - } - - private void TO() throws ParseException { - parser.begin(); - if( !(parser.match("TO") && Space()) ) - throw exception("'TO' expected"); - Spaces(); - parser.success(); - } - - private String SimpleTerm() throws ParseException { - parser.begin(); - String match; - if( parser.match('"') ) { - int start = parser.currentIndex() - 1; - while( !parser.match('"') ) { - if( parser.endOfInput() ) - throw exception("unclosed quotes"); - parser.anyChar(); - checkEscape(); - } - match = parser.textFrom(start); - Spaces(); - } else { - match = Unquoted(NOT_IN_TERM); - } - if( match.length() == 0 ) - throw exception("invalid input"); - return parser.success(match); - } - - private String QueryField() throws ParseException { - parser.begin(); - String match = Field(); - if( match==null || !parser.match(':') ) - return parser.failure((String)null); - Spaces(); - return parser.success(match); - } - - private String Field() throws ParseException { - parser.begin(); - String match = Unquoted(NOT_IN_FIELD); - if( match.length()==0 ) - return parser.failure((String)null); - match = StringFieldParser.escape(this,match); - return parser.success(match); - } - - private String Unquoted(String exclude) throws ParseException { - int start = parser.begin(); - while( parser.noneOf(exclude) ) { - checkEscape(); - } - String match = parser.textFrom(start); - Spaces(); - return parser.success(match); - } - - private void checkEscape() { - if( parser.lastChar() == '\\' ) - parser.anyChar(); - } - - private void Spaces() { - while( Space() ); - } - - private boolean Space() { - return parser.anyOf(" \t\r\n"); - } - - - // sort - - private Sort parseSort() throws ParseException { - Spaces(); - if( parser.endOfInput() ) - return null; - List<SortField> list = new ArrayList<SortField>(); - list.add( SortField() ); - while( !parser.endOfInput() ) { - parser.begin(); - if( !parser.match(',') ) - throw exception("',' expected"); - Spaces(); - parser.success(); - list.add( SortField() ); - } - return new Sort(list.toArray(new SortField[0])); - } - - private SortField SortField() throws ParseException { - parser.begin(); - String field = Field(); - if( field==null ) - throw exception("invalid input"); - boolean reverse = !parser.matchIgnoreCase("asc") && parser.matchIgnoreCase("desc"); - Spaces(); - SortField sf = fieldParser.getSortField(this,field,reverse); - return parser.success(sf); - } - -}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/queryparser/StringFieldParser.java --- a/src/luan/modules/lucene/queryparser/StringFieldParser.java Fri Feb 22 10:12:05 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,113 +0,0 @@ -package luan.modules.lucene.queryparser; - -import java.io.StringReader; -import java.io.IOException; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.SortField; -import org.apache.lucene.index.Term; -import luan.lib.parser.ParseException; - - -public class StringFieldParser implements FieldParser { - public int slop = 0; - public final Analyzer analyzer; - - public StringFieldParser(Analyzer analyzer) { - this.analyzer = analyzer; - } - - @Override public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { - String wildcard = wildcard(qp,query); - if( wildcard != null ) - return new WildcardQuery(new Term(field,wildcard)); - if( query.endsWith("*") && !query.endsWith("\\*") ) - return new PrefixQuery(new Term(field,query.substring(0,query.length()-1))); - query = escape(qp,query); - PhraseQuery pq = new PhraseQuery(); - try { - TokenStream ts = analyzer.tokenStream(field,new StringReader(query)); - CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class); - PositionIncrementAttribute posAttr = ts.addAttribute(PositionIncrementAttribute.class); - ts.reset(); - int pos = -1; - while( ts.incrementToken() ) { - pos += posAttr.getPositionIncrement(); - pq.add( new Term(field,termAttr.toString()), pos ); - } - ts.end(); - ts.close(); - } catch(IOException e) { - throw new RuntimeException(e); - } - Term[] terms = pq.getTerms(); - if( terms.length==1 && pq.getPositions()[0]==0 ) - return new TermQuery(terms[0]); - return pq; - } - - @Override public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { - minQuery = escape(qp,minQuery); - maxQuery = escape(qp,maxQuery); - return TermRangeQuery.newStringRange(field,minQuery,maxQuery,includeMin,includeMax); - } - - static String escape(SaneQueryParser qp,String s) throws ParseException { - final char[] a = s.toCharArray(); - int i, n; - if( a[0] == '"' ) { - if( a[a.length-1] != '"' ) throw new RuntimeException(); - i = 1; - n = a.length - 1; - } else { - i = 0; - n = a.length; - } - StringBuilder sb = new StringBuilder(); - for( ; i<n; i++ ) { - char c = a[i]; - if( c == '\\' ) { - if( ++i == a.length ) - throw qp.exception("ends with '\\'"); - c = a[i]; - } - sb.append(c); - } - return sb.toString(); - } - - private static String wildcard(SaneQueryParser qp,String s) throws ParseException { - final char[] a = s.toCharArray(); - if( a[0] == '"' ) - return null; - boolean hasWildcard = false; - StringBuilder sb = new StringBuilder(); - for( int i=0; i<a.length; i++ ) { - char c = a[i]; - if( c=='?' || c=='*' && i<a.length-1 ) - hasWildcard = true; - if( c == '\\' ) { - if( ++i == a.length ) - throw qp.exception("ends with '\\'"); - c = a[i]; - if( c=='?' || c=='*' ) - sb.append('\\'); - } - sb.append(c); - } - return hasWildcard ? sb.toString() : null; - } - - @Override public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) { - return new SortField( field, SortField.Type.STRING, reverse ); - } - -}
diff -r 7d9a1f8894b0 -r dc2af9d5463b src/luan/modules/lucene/queryparser/SynonymParser.java --- a/src/luan/modules/lucene/queryparser/SynonymParser.java Fri Feb 22 10:12:05 2019 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,43 +0,0 @@ -package luan.modules.lucene.queryparser; - -import java.util.Map; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.SortField; -import luan.lib.parser.ParseException; - - -public class SynonymParser implements FieldParser { - private final FieldParser fp; - private final Map<String,String[]> synonymMap; - - public SynonymParser(FieldParser fp,Map<String,String[]> synonymMap) { - this.fp = fp; - this.synonymMap = synonymMap; - } - - protected String[] getSynonyms(String query) { - return synonymMap.get(query); - } - - public Query getQuery(SaneQueryParser qp,String field,String query) throws ParseException { - String[] synonyms = getSynonyms(query); - if( synonyms == null ) - return fp.getQuery(qp,field,query); - BooleanQuery bq = new BooleanQuery(); - bq.add( fp.getQuery(qp,field,query), BooleanClause.Occur.SHOULD ); - for( String s : synonyms ) { - bq.add( fp.getQuery(qp,field,s), BooleanClause.Occur.SHOULD ); - } - return bq; - } - - public Query getRangeQuery(SaneQueryParser qp,String field,String minQuery,String maxQuery,boolean includeMin,boolean includeMax) throws ParseException { - return fp.getRangeQuery(qp,field,minQuery,maxQuery,includeMin,includeMax); - } - - public SortField getSortField(SaneQueryParser qp,String field,boolean reverse) throws ParseException { - return fp.getSortField(qp,field,reverse); - } -}