1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20
21 import java.lang.Character.UnicodeBlock;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Locale;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.UUID;
31 import java.util.function.Consumer;
32 import java.util.stream.Stream;
33
34 import javax.annotation.PostConstruct;
35 import javax.servlet.http.HttpServletRequest;
36 import javax.servlet.http.HttpSession;
37
38 import org.apache.logging.log4j.LogManager;
39 import org.apache.logging.log4j.Logger;
40 import org.apache.lucene.index.Term;
41 import org.apache.lucene.queryparser.classic.ParseException;
42 import org.apache.lucene.queryparser.classic.QueryParser;
43 import org.apache.lucene.search.BooleanClause;
44 import org.apache.lucene.search.BooleanQuery;
45 import org.apache.lucene.search.BoostQuery;
46 import org.apache.lucene.search.FuzzyQuery;
47 import org.apache.lucene.search.MatchAllDocsQuery;
48 import org.apache.lucene.search.PhraseQuery;
49 import org.apache.lucene.search.PrefixQuery;
50 import org.apache.lucene.search.Query;
51 import org.apache.lucene.search.TermQuery;
52 import org.apache.lucene.search.TermRangeQuery;
53 import org.apache.lucene.search.WildcardQuery;
54 import org.apache.lucene.util.BytesRef;
55 import org.codelibs.core.lang.StringUtil;
56 import org.codelibs.core.misc.Pair;
57 import org.codelibs.fesen.action.search.SearchRequestBuilder;
58 import org.codelibs.fesen.common.unit.Fuzziness;
59 import org.codelibs.fesen.index.query.BoolQueryBuilder;
60 import org.codelibs.fesen.index.query.QueryBuilder;
61 import org.codelibs.fesen.index.query.QueryBuilders;
62 import org.codelibs.fesen.index.query.RangeQueryBuilder;
63 import org.codelibs.fesen.index.query.functionscore.FunctionScoreQueryBuilder.FilterFunctionBuilder;
64 import org.codelibs.fesen.index.query.functionscore.ScoreFunctionBuilder;
65 import org.codelibs.fesen.index.query.functionscore.ScoreFunctionBuilders;
66 import org.codelibs.fesen.search.rescore.RescorerBuilder;
67 import org.codelibs.fesen.search.sort.SortBuilder;
68 import org.codelibs.fesen.search.sort.SortBuilders;
69 import org.codelibs.fesen.search.sort.SortOrder;
70 import org.codelibs.fess.Constants;
71 import org.codelibs.fess.entity.FacetInfo;
72 import org.codelibs.fess.entity.GeoInfo;
73 import org.codelibs.fess.entity.QueryContext;
74 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
75 import org.codelibs.fess.exception.InvalidQueryException;
76 import org.codelibs.fess.mylasta.action.FessUserBean;
77 import org.codelibs.fess.mylasta.direction.FessConfig;
78 import org.codelibs.fess.score.QueryRescorer;
79 import org.codelibs.fess.util.ComponentUtil;
80 import org.dbflute.optional.OptionalThing;
81 import org.lastaflute.core.message.UserMessages;
82 import org.lastaflute.web.util.LaRequestUtil;
83
84 public class QueryHelper {
85 private static final Logger logger = LogManager.getLogger(QueryHelper.class);
86
87 protected static final String PREFERENCE_QUERY = "_query";
88
89 protected static final String ES_SCORE_FIELD = "_score";
90
91 protected static final String SCORE_SORT_VALUE = "score";
92
93 protected static final String SCORE_FIELD = "score";
94
95 protected static final String INURL_FIELD = "inurl";
96
97 protected static final String SITE_FIELD = "site";
98
99 protected Set<String> apiResponseFieldSet;
100
101 protected Set<String> highlightFieldSet = new HashSet<>();
102
103 protected Set<String> notAnalyzedFieldSet;
104
105 protected String[] responseFields;
106
107 protected String[] scrollResponseFields;
108
109 protected String[] cacheResponseFields;
110
111 protected String[] highlightedFields;
112
113 protected String[] searchFields;
114
115 protected String[] facetFields;
116
117 protected String sortPrefix = "sort:";
118
119 protected String[] sortFields;
120
121 protected String additionalQuery;
122
123 protected boolean lowercaseWildcard = true;
124
125 protected SortBuilder<?>[] defaultSortBuilders;
126
127 protected String highlightPrefix = "hl_";
128
129 protected FacetInfo defaultFacetInfo;
130
131 protected GeoInfo defaultGeoInfo;
132
133 protected Map<String, String> fieldBoostMap = new HashMap<>();
134
135 protected List<FilterFunctionBuilder> boostFunctionList = new ArrayList<>();
136
137 protected List<QueryRescorer> queryRescorerList = new ArrayList<>();
138
139 protected List<Pair<String, Float>> additionalDefaultList = new ArrayList<>();
140
141 @PostConstruct
142 public void init() {
143 if (logger.isDebugEnabled()) {
144 logger.debug("Initialize {}", this.getClass().getSimpleName());
145 }
146 final FessConfig fessConfig = ComponentUtil.getFessConfig();
147 if (responseFields == null) {
148 responseFields = fessConfig.getQueryAdditionalResponseFields(
149 SCORE_FIELD,
150 fessConfig.getIndexFieldId(),
151 fessConfig.getIndexFieldDocId(),
152 fessConfig.getIndexFieldBoost(),
153 fessConfig.getIndexFieldContentLength(),
154 fessConfig.getIndexFieldHost(),
155 fessConfig.getIndexFieldSite(),
156 fessConfig.getIndexFieldLastModified(),
157 fessConfig.getIndexFieldTimestamp(),
158 fessConfig.getIndexFieldMimetype(),
159 fessConfig.getIndexFieldFiletype(),
160 fessConfig.getIndexFieldFilename(),
161 fessConfig.getIndexFieldCreated(),
162 fessConfig.getIndexFieldTitle(),
163 fessConfig.getIndexFieldDigest(),
164 fessConfig.getIndexFieldUrl(),
165 fessConfig.getIndexFieldThumbnail(),
166 fessConfig.getIndexFieldClickCount(),
167 fessConfig.getIndexFieldFavoriteCount(),
168 fessConfig.getIndexFieldConfigId(),
169 fessConfig.getIndexFieldLang(),
170 fessConfig.getIndexFieldHasCache());
171 }
172 if (scrollResponseFields == null) {
173 scrollResponseFields = fessConfig.getQueryAdditionalScrollResponseFields(
174 SCORE_FIELD,
175 fessConfig.getIndexFieldId(),
176 fessConfig.getIndexFieldDocId(),
177 fessConfig.getIndexFieldBoost(),
178 fessConfig.getIndexFieldContentLength(),
179 fessConfig.getIndexFieldHost(),
180 fessConfig.getIndexFieldSite(),
181 fessConfig.getIndexFieldLastModified(),
182 fessConfig.getIndexFieldTimestamp(),
183 fessConfig.getIndexFieldMimetype(),
184 fessConfig.getIndexFieldFiletype(),
185 fessConfig.getIndexFieldFilename(),
186 fessConfig.getIndexFieldCreated(),
187 fessConfig.getIndexFieldTitle(),
188 fessConfig.getIndexFieldDigest(),
189 fessConfig.getIndexFieldUrl(),
190 fessConfig.getIndexFieldThumbnail(),
191 fessConfig.getIndexFieldClickCount(),
192 fessConfig.getIndexFieldFavoriteCount(),
193 fessConfig.getIndexFieldConfigId(),
194 fessConfig.getIndexFieldLang(),
195 fessConfig.getIndexFieldHasCache());
196 }
197 if (cacheResponseFields == null) {
198 cacheResponseFields = fessConfig.getQueryAdditionalCacheResponseFields(
199 SCORE_FIELD,
200 fessConfig.getIndexFieldId(),
201 fessConfig.getIndexFieldDocId(),
202 fessConfig.getIndexFieldBoost(),
203 fessConfig.getIndexFieldContentLength(),
204 fessConfig.getIndexFieldHost(),
205 fessConfig.getIndexFieldSite(),
206 fessConfig.getIndexFieldLastModified(),
207 fessConfig.getIndexFieldTimestamp(),
208 fessConfig.getIndexFieldMimetype(),
209 fessConfig.getIndexFieldFiletype(),
210 fessConfig.getIndexFieldFilename(),
211 fessConfig.getIndexFieldCreated(),
212 fessConfig.getIndexFieldTitle(),
213 fessConfig.getIndexFieldDigest(),
214 fessConfig.getIndexFieldUrl(),
215 fessConfig.getIndexFieldClickCount(),
216 fessConfig.getIndexFieldFavoriteCount(),
217 fessConfig.getIndexFieldConfigId(),
218 fessConfig.getIndexFieldLang(),
219 fessConfig.getIndexFieldCache());
220 }
221 if (highlightedFields == null) {
222 highlightedFields = fessConfig.getQueryAdditionalHighlightedFields(
223 fessConfig.getIndexFieldContent());
224 }
225 if (searchFields == null) {
226 searchFields = fessConfig.getQueryAdditionalSearchFields(
227 INURL_FIELD,
228 fessConfig.getIndexFieldUrl(),
229 fessConfig.getIndexFieldDocId(),
230 fessConfig.getIndexFieldHost(),
231 fessConfig.getIndexFieldSite(),
232 fessConfig.getIndexFieldTitle(),
233 fessConfig.getIndexFieldContent(),
234 fessConfig.getIndexFieldContentLength(),
235 fessConfig.getIndexFieldLastModified(),
236 fessConfig.getIndexFieldTimestamp(),
237 fessConfig.getIndexFieldMimetype(),
238 fessConfig.getIndexFieldFiletype(),
239 fessConfig.getIndexFieldFilename(),
240 fessConfig.getIndexFieldLabel(),
241 fessConfig.getIndexFieldSegment(),
242 fessConfig.getIndexFieldAnchor(),
243 fessConfig.getIndexFieldClickCount(),
244 fessConfig.getIndexFieldFavoriteCount(),
245 fessConfig.getIndexFieldLang());
246 }
247 if (facetFields == null) {
248 facetFields = fessConfig.getQueryAdditionalFacetFields(
249 fessConfig.getIndexFieldUrl(),
250 fessConfig.getIndexFieldHost(),
251 fessConfig.getIndexFieldTitle(),
252 fessConfig.getIndexFieldContent(),
253 fessConfig.getIndexFieldContentLength(),
254 fessConfig.getIndexFieldLastModified(),
255 fessConfig.getIndexFieldTimestamp(),
256 fessConfig.getIndexFieldMimetype(),
257 fessConfig.getIndexFieldFiletype(),
258 fessConfig.getIndexFieldLabel(),
259 fessConfig.getIndexFieldSegment());
260 }
261 if (sortFields == null) {
262 sortFields = fessConfig.getQueryAdditionalSortFields(
263 SCORE_SORT_VALUE,
264 fessConfig.getIndexFieldFilename(),
265 fessConfig.getIndexFieldCreated(),
266 fessConfig.getIndexFieldContentLength(),
267 fessConfig.getIndexFieldLastModified(),
268 fessConfig.getIndexFieldTimestamp(),
269 fessConfig.getIndexFieldClickCount(),
270 fessConfig.getIndexFieldFavoriteCount());
271 }
272 if (apiResponseFieldSet == null) {
273 setApiResponseFields(fessConfig.getQueryAdditionalApiResponseFields(
274 fessConfig.getResponseFieldContentDescription(),
275 fessConfig.getResponseFieldContentTitle(),
276 fessConfig.getResponseFieldSitePath(),
277 fessConfig.getResponseFieldUrlLink(),
278 fessConfig.getIndexFieldId(),
279 fessConfig.getIndexFieldDocId(),
280 fessConfig.getIndexFieldBoost(),
281 fessConfig.getIndexFieldContentLength(),
282 fessConfig.getIndexFieldHost(),
283 fessConfig.getIndexFieldSite(),
284 fessConfig.getIndexFieldLastModified(),
285 fessConfig.getIndexFieldTimestamp(),
286 fessConfig.getIndexFieldMimetype(),
287 fessConfig.getIndexFieldFiletype(),
288 fessConfig.getIndexFieldFilename(),
289 fessConfig.getIndexFieldCreated(),
290 fessConfig.getIndexFieldTitle(),
291 fessConfig.getIndexFieldDigest(),
292 fessConfig.getIndexFieldUrl()));
293 }
294 if (notAnalyzedFieldSet == null) {
295 setNotAnalyzedFields(fessConfig.getQueryAdditionalNotAnalyzedFields(
296 fessConfig.getIndexFieldAnchor(),
297 fessConfig.getIndexFieldBoost(),
298 fessConfig.getIndexFieldClickCount(),
299 fessConfig.getIndexFieldConfigId(),
300 fessConfig.getIndexFieldContentLength(),
301 fessConfig.getIndexFieldCreated(),
302 fessConfig.getIndexFieldDocId(),
303 fessConfig.getIndexFieldExpires(),
304 fessConfig.getIndexFieldFavoriteCount(),
305 fessConfig.getIndexFieldFiletype(),
306 fessConfig.getIndexFieldFilename(),
307 fessConfig.getIndexFieldHasCache(),
308 fessConfig.getIndexFieldHost(),
309 fessConfig.getIndexFieldId(),
310 fessConfig.getIndexFieldLabel(),
311 fessConfig.getIndexFieldLang(),
312 fessConfig.getIndexFieldLastModified(),
313 fessConfig.getIndexFieldMimetype(),
314 fessConfig.getIndexFieldParentId(),
315 fessConfig.getIndexFieldPrimaryTerm(),
316 fessConfig.getIndexFieldRole(),
317 fessConfig.getIndexFieldSegment(),
318 fessConfig.getIndexFieldSeqNo(),
319 fessConfig.getIndexFieldSite(),
320 fessConfig.getIndexFieldTimestamp(),
321 fessConfig.getIndexFieldUrl(),
322 fessConfig.getIndexFieldVersion()));
323 }
324 split(fessConfig.getQueryAdditionalAnalyzedFields(), ",")
325 .of(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).forEach(s -> notAnalyzedFieldSet.remove(s)));
326 split(fessConfig.getQueryAdditionalDefaultFields(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).map(s -> {
327 final Pair<String, Float> pair = new Pair<>();
328 final String[] values = s.split(":");
329 if (values.length == 1) {
330 pair.setFirst(values[0].trim());
331 pair.setSecond(1.0f);
332 } else if (values.length > 1) {
333 pair.setFirst(values[0]);
334 pair.setSecond(Float.parseFloat(values[1]));
335 } else {
336 return null;
337 }
338 return pair;
339 }).forEach(additionalDefaultList::add));
340 }
341
342 public QueryContext build(final SearchRequestType searchRequestType, final String query, final Consumer<QueryContext> context) {
343 String q;
344 if (additionalQuery != null && StringUtil.isNotBlank(query)) {
345 q = query + " " + additionalQuery;
346 } else {
347 q = query;
348 }
349
350 final QueryContext queryContext = new QueryContext(q, true);
351 buildBaseQuery(queryContext, context);
352 buildBoostQuery(queryContext);
353 buildRoleQuery(queryContext, searchRequestType);
354 buildVirtualHostQuery(queryContext, searchRequestType);
355
356 if (!queryContext.hasSorts() && defaultSortBuilders != null) {
357 queryContext.addSorts(defaultSortBuilders);
358 }
359 return queryContext;
360 }
361
362 protected void buildVirtualHostQuery(final QueryContext queryContext, final SearchRequestType searchRequestType) {
363 switch (searchRequestType) {
364 case ADMIN_SEARCH:
365
366 break;
367 default:
368 final String key = ComponentUtil.getVirtualHostHelper().getVirtualHostKey();
369 if (StringUtil.isNotBlank(key)) {
370 queryContext.addQuery(boolQuery -> {
371 boolQuery.filter(QueryBuilders.termQuery(ComponentUtil.getFessConfig().getIndexFieldVirtualHost(), key));
372 });
373 }
374 break;
375 }
376 }
377
378 protected void buildRoleQuery(final QueryContext queryContext, final SearchRequestType searchRequestType) {
379 if (queryContext.roleQueryEnabled()) {
380 final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(searchRequestType);
381 if (!roleSet.isEmpty()) {
382 queryContext.addQuery(boolQuery -> buildRoleQuery(roleSet, boolQuery));
383 }
384 }
385 }
386
387 public void buildRoleQuery(final Set<String> roleSet, final BoolQueryBuilder boolQuery) {
388 final BoolQueryBuilder roleQuery = QueryBuilders.boolQuery();
389 final FessConfig fessConfig = ComponentUtil.getFessConfig();
390 final String roleField = fessConfig.getIndexFieldRole();
391 roleSet.stream().forEach(name -> roleQuery.should(QueryBuilders.termQuery(roleField, name)));
392 final String deniedPrefix = fessConfig.getRoleSearchDeniedPrefix();
393 roleSet.stream().forEach(name -> roleQuery.mustNot(QueryBuilders.termQuery(roleField, deniedPrefix + name)));
394 boolQuery.filter(roleQuery);
395 }
396
397 protected void buildBoostQuery(final QueryContext queryContext) {
398 queryContext.addFunctionScore(list -> {
399 list.add(new FilterFunctionBuilder(
400 ScoreFunctionBuilders.fieldValueFactorFunction(ComponentUtil.getFessConfig().getIndexFieldBoost())));
401 ComponentUtil.getKeyMatchHelper().buildQuery(queryContext.getDefaultKeyword(), list);
402 list.addAll(boostFunctionList);
403 });
404 }
405
406 public void buildBaseQuery(final QueryContext queryContext, final Consumer<QueryContext> context) {
407 try {
408 final Query query = getQueryParser().parse(queryContext.getQueryString());
409 final QueryBuilder queryBuilder = convertQuery(queryContext, query, 1.0f);
410 if (queryBuilder != null) {
411 queryContext.setQueryBuilder(queryBuilder);
412 } else {
413 queryContext.setQueryBuilder(QueryBuilders.matchAllQuery());
414 }
415
416 context.accept(queryContext);
417 } catch (final ParseException e) {
418 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryParseError(UserMessages.GLOBAL_PROPERTY_KEY),
419 "Invalid query: " + queryContext.getQueryString(), e);
420 }
421 }
422
423 protected QueryParser getQueryParser() {
424 return ComponentUtil.getQueryParser();
425 }
426
427 protected QueryBuilder convertQuery(final QueryContext context, final Query query, final float boost) {
428 if (query instanceof TermQuery) {
429 return convertTermQuery(context, (TermQuery) query, boost);
430 }
431 if (query instanceof TermRangeQuery) {
432 return convertTermRangeQuery(context, (TermRangeQuery) query, boost);
433 }
434 if (query instanceof PhraseQuery) {
435 return convertPhraseQuery(context, (PhraseQuery) query, boost);
436 } else if (query instanceof FuzzyQuery) {
437 return convertFuzzyQuery(context, (FuzzyQuery) query, boost);
438 } else if (query instanceof PrefixQuery) {
439 return convertPrefixQuery(context, (PrefixQuery) query, boost);
440 } else if (query instanceof WildcardQuery) {
441 return convertWildcardQuery(context, (WildcardQuery) query, boost);
442 } else if (query instanceof BooleanQuery) {
443 final BooleanQuery booleanQuery = (BooleanQuery) query;
444 return convertBooleanQuery(context, booleanQuery, boost);
445 } else if (query instanceof MatchAllDocsQuery) {
446 return QueryBuilders.matchAllQuery();
447 } else if (query instanceof BoostQuery) {
448 final BoostQuery boostQuery = (BoostQuery) query;
449 return convertQuery(context, boostQuery.getQuery(), boostQuery.getBoost());
450 }
451 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnknown(UserMessages.GLOBAL_PROPERTY_KEY),
452 "Unknown q: " + query.getClass() + " => " + query);
453 }
454
455 protected QueryBuilder convertBooleanQuery(final QueryContext context, final BooleanQuery booleanQuery, final float boost) {
456 final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
457 for (final BooleanClause clause : booleanQuery.clauses()) {
458 final QueryBuilder queryBuilder = convertQuery(context, clause.getQuery(), boost);
459 if (queryBuilder != null) {
460 switch (clause.getOccur()) {
461 case MUST:
462 boolQuery.must(queryBuilder);
463 break;
464 case SHOULD:
465 boolQuery.should(queryBuilder);
466 break;
467 case MUST_NOT:
468 boolQuery.mustNot(queryBuilder);
469 break;
470 default:
471 break;
472 }
473 }
474 }
475 if (boolQuery.hasClauses()) {
476 return boolQuery;
477 }
478 return null;
479 }
480
481 protected String toLowercaseWildcard(final String value) {
482 if (lowercaseWildcard) {
483 return value.toLowerCase(Locale.ROOT);
484 }
485 return value;
486 }
487
488 protected String getSearchField(final QueryContext context, final String field) {
489 if (Constants.DEFAULT_FIELD.equals(field) && context.getDefaultField() != null) {
490 return context.getDefaultField();
491 }
492 return field;
493 }
494
495 protected QueryBuilder convertWildcardQuery(final QueryContext context, final WildcardQuery wildcardQuery, final float boost) {
496 final String field = getSearchField(context, wildcardQuery.getField());
497 if (Constants.DEFAULT_FIELD.equals(field)) {
498 context.addFieldLog(field, wildcardQuery.getTerm().text());
499 return buildDefaultQueryBuilder(
500 (f, b) -> QueryBuilders.wildcardQuery(f, toLowercaseWildcard(wildcardQuery.getTerm().text())).boost(b * boost));
501 }
502 if (isSearchField(field)) {
503 context.addFieldLog(field, wildcardQuery.getTerm().text());
504 return QueryBuilders.wildcardQuery(field, toLowercaseWildcard(wildcardQuery.getTerm().text())).boost(boost);
505 }
506 final String query = wildcardQuery.getTerm().toString();
507 final String origQuery = toLowercaseWildcard(query);
508 context.addFieldLog(Constants.DEFAULT_FIELD, query);
509 context.addHighlightedQuery(origQuery);
510 return buildDefaultQueryBuilder((f, b) -> QueryBuilders.wildcardQuery(f, origQuery).boost(b * boost));
511 }
512
513 protected QueryBuilder convertPrefixQuery(final QueryContext context, final PrefixQuery prefixQuery, final float boost) {
514 final FessConfig fessConfig = ComponentUtil.getFessConfig();
515 final String field = getSearchField(context, prefixQuery.getField());
516 if (Constants.DEFAULT_FIELD.equals(field)) {
517 context.addFieldLog(field, prefixQuery.getPrefix().text());
518 return buildDefaultQueryBuilder(
519 (f, b) -> QueryBuilders.matchPhrasePrefixQuery(f, toLowercaseWildcard(prefixQuery.getPrefix().text())).boost(b * boost)
520 .maxExpansions(fessConfig.getQueryPrefixExpansionsAsInteger()).slop(fessConfig.getQueryPrefixSlopAsInteger()));
521 }
522 if (!isSearchField(field)) {
523 final String query = prefixQuery.getPrefix().toString();
524 final String origQuery = toLowercaseWildcard(query);
525 context.addFieldLog(Constants.DEFAULT_FIELD, query);
526 context.addHighlightedQuery(origQuery);
527 return buildDefaultQueryBuilder((f, b) -> QueryBuilders.matchPhrasePrefixQuery(f, origQuery).boost(b * boost)
528 .maxExpansions(fessConfig.getQueryPrefixExpansionsAsInteger()).slop(fessConfig.getQueryPrefixSlopAsInteger()));
529 }
530 context.addFieldLog(field, prefixQuery.getPrefix().text());
531 if (notAnalyzedFieldSet.contains(field)) {
532 return QueryBuilders.prefixQuery(field, toLowercaseWildcard(prefixQuery.getPrefix().text())).boost(boost);
533 } else {
534 return QueryBuilders.matchPhrasePrefixQuery(field, toLowercaseWildcard(prefixQuery.getPrefix().text())).boost(boost)
535 .maxExpansions(fessConfig.getQueryPrefixExpansionsAsInteger()).slop(fessConfig.getQueryPrefixSlopAsInteger());
536 }
537 }
538
539 protected QueryBuilder convertFuzzyQuery(final QueryContext context, final FuzzyQuery fuzzyQuery, final float boost) {
540 final FessConfig fessConfig = ComponentUtil.getFessConfig();
541 final Term term = fuzzyQuery.getTerm();
542 final String field = getSearchField(context, term.field());
543
544 if (Constants.DEFAULT_FIELD.equals(field)) {
545 context.addFieldLog(field, term.text());
546 return buildDefaultQueryBuilder(
547 (f, b) -> QueryBuilders.fuzzyQuery(f, term.text()).fuzziness(Fuzziness.fromEdits(fuzzyQuery.getMaxEdits()))
548 .boost(b * boost).maxExpansions(fessConfig.getQueryFuzzyExpansionsAsInteger())
549 .prefixLength(fessConfig.getQueryFuzzyPrefixLengthAsInteger())
550 .transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryFuzzyTranspositions())));
551 }
552 if (isSearchField(field)) {
553 context.addFieldLog(field, term.text());
554 return QueryBuilders.fuzzyQuery(field, term.text()).boost(boost).fuzziness(Fuzziness.fromEdits(fuzzyQuery.getMaxEdits()))
555 .maxExpansions(fessConfig.getQueryFuzzyExpansionsAsInteger())
556 .prefixLength(fessConfig.getQueryFuzzyPrefixLengthAsInteger())
557 .transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryFuzzyTranspositions()));
558 }
559 final String origQuery = fuzzyQuery.toString();
560 context.addFieldLog(Constants.DEFAULT_FIELD, origQuery);
561 context.addHighlightedQuery(origQuery);
562 return buildDefaultQueryBuilder((f, b) -> QueryBuilders.fuzzyQuery(f, origQuery)
563 .fuzziness(Fuzziness.fromEdits(fuzzyQuery.getMaxEdits())).boost(b * boost)
564 .maxExpansions(fessConfig.getQueryFuzzyExpansionsAsInteger()).prefixLength(fessConfig.getQueryFuzzyPrefixLengthAsInteger())
565 .transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryFuzzyTranspositions())));
566 }
567
568 protected QueryBuilder convertTermRangeQuery(final QueryContext context, final TermRangeQuery termRangeQuery, final float boost) {
569 final String field = getSearchField(context, termRangeQuery.getField());
570 if (!isSearchField(field)) {
571 final String origQuery = termRangeQuery.toString();
572 context.addFieldLog(Constants.DEFAULT_FIELD, origQuery);
573 context.addHighlightedQuery(origQuery);
574 return buildDefaultQueryBuilder((f, b) -> QueryBuilders.matchPhraseQuery(f, origQuery).boost(b));
575 }
576 context.addFieldLog(field, termRangeQuery.toString(field));
577 final RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(field);
578 final BytesRef min = termRangeQuery.getLowerTerm();
579 if (min != null) {
580 if (termRangeQuery.includesLower()) {
581 rangeQuery.gte(min.utf8ToString());
582 } else {
583 rangeQuery.gt(min.utf8ToString());
584 }
585 }
586 final BytesRef max = termRangeQuery.getUpperTerm();
587 if (max != null) {
588 if (termRangeQuery.includesUpper()) {
589 rangeQuery.lte(max.utf8ToString());
590 } else {
591 rangeQuery.lt(max.utf8ToString());
592 }
593 }
594 rangeQuery.boost(boost);
595 return rangeQuery;
596 }
597
598 protected QueryBuilder buildMatchPhraseQuery(final String f, final String text) {
599 final FessConfig fessConfig = ComponentUtil.getFessConfig();
600 if (text == null || text.length() != 1
601 || (!fessConfig.getIndexFieldTitle().equals(f) && !fessConfig.getIndexFieldContent().equals(f))) {
602 return QueryBuilders.matchPhraseQuery(f, text);
603 }
604
605 final UnicodeBlock block = UnicodeBlock.of(text.codePointAt(0));
606 if (block == UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS
607 || block == UnicodeBlock.HIRAGANA
608 || block == UnicodeBlock.KATAKANA
609 || block == UnicodeBlock.HANGUL_SYLLABLES
610 ) {
611 return QueryBuilders.prefixQuery(f, text);
612 }
613 return QueryBuilders.matchPhraseQuery(f, text);
614 }
615
616 protected QueryBuilder convertTermQuery(final QueryContext context, final TermQuery termQuery, final float boost) {
617 final String field = getSearchField(context, termQuery.getTerm().field());
618 final String text = termQuery.getTerm().text();
619 final FessConfig fessConfig = ComponentUtil.getFessConfig();
620 if (fessConfig.getQueryReplaceTermWithPrefixQueryAsBoolean() && text.length() > 1 && text.endsWith("*")) {
621 return convertPrefixQuery(context, new PrefixQuery(new Term(field, text.substring(0, text.length() - 1))), boost);
622 }
623 if (Constants.DEFAULT_FIELD.equals(field)) {
624 context.addFieldLog(field, text);
625 context.addHighlightedQuery(text);
626 return buildDefaultTermQueryBuilder(boost, text);
627 }
628 if ("sort".equals(field)) {
629 split(text, ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(t -> {
630 final String[] values = t.split("\\.");
631 if (values.length > 2) {
632 throw new InvalidQueryException(
633 messages -> messages.addErrorsInvalidQuerySortValue(UserMessages.GLOBAL_PROPERTY_KEY, text),
634 "Invalid sort field: " + termQuery);
635 }
636 final String sortField = values[0];
637 if (!isSortField(sortField)) {
638 throw new InvalidQueryException(
639 messages -> messages.addErrorsInvalidQueryUnsupportedSortField(UserMessages.GLOBAL_PROPERTY_KEY, sortField),
640 "Unsupported sort field: " + termQuery);
641 }
642 SortOrder sortOrder;
643 if (values.length == 2) {
644 sortOrder = SortOrder.DESC.toString().equalsIgnoreCase(values[1]) ? SortOrder.DESC : SortOrder.ASC;
645 if (sortOrder == null) {
646 throw new InvalidQueryException(
647 messages -> messages.addErrorsInvalidQueryUnsupportedSortOrder(UserMessages.GLOBAL_PROPERTY_KEY, values[1]),
648 "Invalid sort order: " + termQuery);
649 }
650 } else {
651 sortOrder = SortOrder.ASC;
652 }
653 context.addSorts(createFieldSortBuilder(sortField, sortOrder));
654 }));
655 return null;
656 } else if (INURL_FIELD.equals(field) || (StringUtil.equals(field, context.getDefaultField())
657 && fessConfig.getIndexFieldUrl().equals(context.getDefaultField()))) {
658 return QueryBuilders.wildcardQuery(fessConfig.getIndexFieldUrl(), "*" + text + "*").boost(boost);
659 } else if (SITE_FIELD.equals(field)) {
660 return convertSiteQuery(context, text, boost);
661 } else if (isSearchField(field)) {
662 context.addFieldLog(field, text);
663 context.addHighlightedQuery(text);
664 if (notAnalyzedFieldSet.contains(field)) {
665 return QueryBuilders.termQuery(field, text).boost(boost);
666 } else {
667 return buildMatchPhraseQuery(field, text).boost(boost);
668 }
669 } else {
670 final String origQuery = termQuery.toString();
671 context.addFieldLog(Constants.DEFAULT_FIELD, origQuery);
672 context.addHighlightedQuery(origQuery);
673 return buildDefaultQueryBuilder((f, b) -> buildMatchPhraseQuery(f, origQuery).boost(b * boost));
674 }
675 }
676
677 protected QueryBuilder convertSiteQuery(final QueryContext context, final String text, final float boost) {
678 return QueryBuilders.prefixQuery(ComponentUtil.getFessConfig().getIndexFieldSite(), text).boost(boost);
679 }
680
681 protected QueryBuilder convertPhraseQuery(final QueryContext context, final PhraseQuery query, final float boost) {
682 final Term[] terms = query.getTerms();
683 if (terms.length == 0) {
684 throw new InvalidQueryException(messages -> messages.addErrorsInvalidQueryUnknown(UserMessages.GLOBAL_PROPERTY_KEY),
685 "Unknown phrase query: " + query);
686 }
687 final String field = terms[0].field();
688 final String[] texts = stream(terms).get(stream -> stream.map(Term::text).toArray(n -> new String[n]));
689 final String text = String.join(" ", texts);
690 context.addFieldLog(field, text);
691 stream(texts).of(stream -> stream.forEach(t -> context.addHighlightedQuery(t)));
692 return buildDefaultQueryBuilder((f, b) -> buildMatchPhraseQuery(f, text).boost(b * boost));
693 }
694
695 protected boolean isSearchField(final String field) {
696 for (final String searchField : searchFields) {
697 if (searchField.equals(field)) {
698 return true;
699 }
700 }
701 return false;
702 }
703
704 protected QueryBuilder buildDefaultTermQueryBuilder(final float boost, final String text) {
705 final BoolQueryBuilder boolQuery = buildDefaultQueryBuilder((f, b) -> buildMatchPhraseQuery(f, text).boost(b * boost));
706 final FessConfig fessConfig = ComponentUtil.getFessConfig();
707 final Integer fuzzyMinLength = fessConfig.getQueryBoostFuzzyMinLengthAsInteger();
708 if (fuzzyMinLength >= 0 && text.length() >= fuzzyMinLength) {
709 boolQuery.should(QueryBuilders.fuzzyQuery(fessConfig.getIndexFieldTitle(), text)
710 .boost(fessConfig.getQueryBoostFuzzyTitleAsDecimal().floatValue())
711 .prefixLength(fessConfig.getQueryBoostFuzzyTitlePrefixLengthAsInteger())
712 .transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryBoostFuzzyTitleTranspositions()))
713 .fuzziness(Fuzziness.build(fessConfig.getQueryBoostFuzzyTitleFuzziness()))
714 .maxExpansions(fessConfig.getQueryBoostFuzzyTitleExpansionsAsInteger()));
715 boolQuery.should(QueryBuilders.fuzzyQuery(fessConfig.getIndexFieldContent(), text)
716 .prefixLength(fessConfig.getQueryBoostFuzzyContentPrefixLengthAsInteger())
717 .transpositions(Constants.TRUE.equalsIgnoreCase(fessConfig.getQueryBoostFuzzyContentTranspositions()))
718 .boost(fessConfig.getQueryBoostFuzzyContentAsDecimal().floatValue())
719 .fuzziness(Fuzziness.build(fessConfig.getQueryBoostFuzzyContentFuzziness()))
720 .maxExpansions(fessConfig.getQueryBoostFuzzyContentExpansionsAsInteger()));
721 }
722 return boolQuery;
723 }
724
725 protected BoolQueryBuilder buildDefaultQueryBuilder(final DefaultQueryBuilderFunction builder) {
726 final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
727 final FessConfig fessConfig = ComponentUtil.getFessConfig();
728 boolQuery.should(builder.apply(fessConfig.getIndexFieldTitle(), fessConfig.getQueryBoostTitleAsDecimal().floatValue()));
729 boolQuery.should(builder.apply(fessConfig.getIndexFieldContent(), fessConfig.getQueryBoostContentAsDecimal().floatValue()));
730 final float importantContentBoost = fessConfig.getQueryBoostImportantContentAsDecimal().floatValue();
731 if (importantContentBoost >= 0.0f) {
732 boolQuery.should(builder.apply(fessConfig.getIndexFieldImportantContent(), importantContentBoost));
733 }
734 final float importantContantLangBoost = fessConfig.getQueryBoostImportantContentLangAsDecimal().floatValue();
735 getQueryLanguages().ifPresent(langs -> stream(langs).of(stream -> stream.forEach(lang -> {
736 boolQuery.should(
737 builder.apply(fessConfig.getIndexFieldTitle() + "_" + lang, fessConfig.getQueryBoostTitleLangAsDecimal().floatValue()));
738 boolQuery.should(builder.apply(fessConfig.getIndexFieldContent() + "_" + lang,
739 fessConfig.getQueryBoostContentLangAsDecimal().floatValue()));
740 if (importantContantLangBoost >= 0.0f) {
741 boolQuery.should(builder.apply(fessConfig.getIndexFieldImportantContent() + "_" + lang, importantContantLangBoost));
742 }
743 })));
744 additionalDefaultList.stream().forEach(f -> {
745 final QueryBuilder query = builder.apply(f.getFirst(), f.getSecond());
746 boolQuery.should(query);
747 });
748 return boolQuery;
749 }
750
751 interface DefaultQueryBuilderFunction {
752 QueryBuilder apply(String field, float boost);
753 }
754
755 protected OptionalThing<String[]> getQueryLanguages() {
756 return LaRequestUtil.getOptionalRequest().map(request -> ComponentUtil.getFessConfig().getQueryLanguages(request.getLocales(),
757 (String[]) request.getAttribute(Constants.REQUEST_LANGUAGES)));
758 }
759
760 protected boolean isSortField(final String field) {
761 for (final String f : sortFields) {
762 if (f.equals(field)) {
763 return true;
764 }
765 }
766 return false;
767 }
768
769 public boolean isFacetField(final String field) {
770 if (StringUtil.isBlank(field)) {
771 return false;
772 }
773 boolean flag = false;
774 for (final String f : facetFields) {
775 if (field.equals(f)) {
776 flag = true;
777 }
778 }
779 return flag;
780 }
781
782 public boolean isFacetSortValue(final String sort) {
783 return "count".equals(sort) || "index".equals(sort);
784 }
785
786 public void setApiResponseFields(final String[] fields) {
787 apiResponseFieldSet = new HashSet<>();
788 Collections.addAll(apiResponseFieldSet, fields);
789 }
790
791 public void setNotAnalyzedFields(final String[] fields) {
792 notAnalyzedFieldSet = new HashSet<>();
793 Collections.addAll(notAnalyzedFieldSet, fields);
794 }
795
796 public boolean isApiResponseField(final String field) {
797 return apiResponseFieldSet.contains(field);
798 }
799
800 public void processSearchPreference(final SearchRequestBuilder searchRequestBuilder, final OptionalThing<FessUserBean> userBean,
801 final String query) {
802 userBean.map(user -> {
803 if (user.hasRoles(ComponentUtil.getFessConfig().getAuthenticationAdminRolesAsArray())) {
804 return Constants.SEARCH_PREFERENCE_LOCAL;
805 }
806 return user.getUserId();
807 }).ifPresent(p -> searchRequestBuilder.setPreference(p)).orElse(() -> LaRequestUtil.getOptionalRequest().map(r -> {
808 final HttpSession session = r.getSession(false);
809 if (session != null) {
810 return session.getId();
811 }
812 final String preference = r.getParameter("preference");
813 if (preference != null) {
814 return Integer.toString(preference.hashCode());
815 }
816 final Object accessType = r.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
817 if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {
818 return processJsonSearchPreference(r, query);
819 }
820 if (Constants.SEARCH_LOG_ACCESS_TYPE_GSA.equals(accessType)) {
821 return processGsaSearchPreference(r, query);
822 }
823 return null;
824 }).ifPresent(p -> searchRequestBuilder.setPreference(p)));
825 }
826
827 protected String processJsonSearchPreference(final HttpServletRequest req, final String query) {
828 final String pref = ComponentUtil.getFessConfig().getQueryJsonDefaultPreference();
829 if (PREFERENCE_QUERY.equals(pref)) {
830 return Integer.toString(query.hashCode());
831 }
832 if (StringUtil.isNotBlank(pref)) {
833 return pref;
834 }
835 return null;
836 }
837
838 protected String processGsaSearchPreference(final HttpServletRequest req, final String query) {
839 final String pref = ComponentUtil.getFessConfig().getQueryGsaDefaultPreference();
840 if (PREFERENCE_QUERY.equals(pref)) {
841 return Integer.toString(query.hashCode());
842 }
843 if (StringUtil.isNotBlank(pref)) {
844 return pref;
845 }
846 return null;
847 }
848
849
850
851
852 public String[] getResponseFields() {
853 return responseFields;
854 }
855
856
857
858
859 public void setResponseFields(final String[] responseFields) {
860 this.responseFields = responseFields;
861 }
862
863 public String[] getScrollResponseFields() {
864 return scrollResponseFields;
865 }
866
867 public void setScrollResponseFields(final String[] scrollResponseFields) {
868 this.scrollResponseFields = scrollResponseFields;
869 }
870
871 public String[] getCacheResponseFields() {
872 return cacheResponseFields;
873 }
874
875 public void setCacheResponseFields(final String[] cacheResponseFields) {
876 this.cacheResponseFields = cacheResponseFields;
877 }
878
879
880
881
882 public String[] getHighlightedFields() {
883 return highlightedFields;
884 }
885
886
887
888
889 public void setHighlightedFields(final String[] highlightedFields) {
890 this.highlightedFields = highlightedFields;
891 }
892
893 public void highlightedFields(final Consumer<Stream<String>> stream) {
894 stream(highlightedFields).of(stream);
895 }
896
897
898
899
900 public String[] getSearchFields() {
901 return searchFields;
902 }
903
904
905
906
907 public void setSearchFields(final String[] supportedFields) {
908 searchFields = supportedFields;
909 }
910
911
912
913
914 public String[] getFacetFields() {
915 return facetFields;
916 }
917
918
919
920
921 public void setFacetFields(final String[] facetFields) {
922 this.facetFields = facetFields;
923 }
924
925
926
927
928 public String getSortPrefix() {
929 return sortPrefix;
930 }
931
932
933
934
935 public void setSortPrefix(final String sortPrefix) {
936 this.sortPrefix = sortPrefix;
937 }
938
939
940
941
942 public String[] getSortFields() {
943 return sortFields;
944 }
945
946
947
948
949 public void setSortFields(final String[] sortFields) {
950 this.sortFields = sortFields;
951 }
952
953 public void addHighlightField(final String field) {
954 highlightFieldSet.add(field);
955 }
956
957
958
959
960 public String getAdditionalQuery() {
961 return additionalQuery;
962 }
963
964
965
966
967 public void setAdditionalQuery(final String additionalQuery) {
968 this.additionalQuery = additionalQuery;
969 }
970
971 public void addDefaultSort(final String fieldName, final String order) {
972 final List<SortBuilder<?>> list = new ArrayList<>();
973 if (defaultSortBuilders != null) {
974 stream(defaultSortBuilders).of(stream -> stream.forEach(builder -> list.add(builder)));
975 }
976 list.add(createFieldSortBuilder(fieldName, SortOrder.DESC.toString().equalsIgnoreCase(order) ? SortOrder.DESC : SortOrder.ASC));
977 defaultSortBuilders = list.toArray(new SortBuilder[list.size()]);
978 }
979
980 protected SortBuilder<?> createFieldSortBuilder(final String field, final SortOrder order) {
981 if (SCORE_FIELD.equals(field) || ES_SCORE_FIELD.equals(field)) {
982 return SortBuilders.scoreSort().order(order);
983 }
984 return SortBuilders.fieldSort(field).order(order);
985 }
986
987 public void setHighlightPrefix(final String highlightPrefix) {
988 this.highlightPrefix = highlightPrefix;
989 }
990
991 public String getHighlightPrefix() {
992 return highlightPrefix;
993 }
994
995 public FacetInfo getDefaultFacetInfo() {
996 return defaultFacetInfo;
997 }
998
999 public void setDefaultFacetInfo(final FacetInfo defaultFacetInfo) {
1000 this.defaultFacetInfo = defaultFacetInfo;
1001 }
1002
1003 public GeoInfo getDefaultGeoInfo() {
1004 return defaultGeoInfo;
1005 }
1006
1007 public void setDefaultGeoInfo(final GeoInfo defaultGeoInfo) {
1008 this.defaultGeoInfo = defaultGeoInfo;
1009 }
1010
1011 public String generateId() {
1012 return UUID.randomUUID().toString().replace("-", StringUtil.EMPTY);
1013 }
1014
1015 public void setLowercaseWildcard(final boolean lowercaseWildcard) {
1016 this.lowercaseWildcard = lowercaseWildcard;
1017 }
1018
1019 public void addBoostFunction(final ScoreFunctionBuilder<?> scoreFunction) {
1020 boostFunctionList.add(new FilterFunctionBuilder(scoreFunction));
1021 }
1022
1023 public void addBoostFunction(final QueryBuilder filter, final ScoreFunctionBuilder<?> scoreFunction) {
1024 boostFunctionList.add(new FilterFunctionBuilder(filter, scoreFunction));
1025 }
1026
1027 public RescorerBuilder<?>[] getRescorers(final Map<String, Object> params) {
1028 return queryRescorerList.stream().map(r -> r.evaluate(params)).filter(b -> b != null).toArray(n -> new RescorerBuilder<?>[n]);
1029 }
1030
1031 public void addQueryRescorer(final QueryRescorer rescorer) {
1032 queryRescorerList.add(rescorer);
1033 }
1034 }