1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import java.text.NumberFormat;
19 import java.util.Enumeration;
20 import java.util.HashMap;
21 import java.util.HashSet;
22 import java.util.List;
23 import java.util.Locale;
24 import java.util.Map;
25 import java.util.Map.Entry;
26 import java.util.Set;
27 import java.util.concurrent.ExecutionException;
28 import java.util.function.Consumer;
29 import java.util.stream.Collectors;
30
31 import javax.servlet.http.HttpServletRequest;
32
33 import org.apache.logging.log4j.LogManager;
34 import org.apache.logging.log4j.Logger;
35 import org.codelibs.core.lang.StringUtil;
36 import org.codelibs.fesen.FesenException;
37 import org.codelibs.fesen.action.DocWriteResponse.Result;
38 import org.codelibs.fesen.action.bulk.BulkRequestBuilder;
39 import org.codelibs.fesen.action.bulk.BulkResponse;
40 import org.codelibs.fesen.action.update.UpdateRequestBuilder;
41 import org.codelibs.fesen.action.update.UpdateResponse;
42 import org.codelibs.fesen.common.document.DocumentField;
43 import org.codelibs.fesen.index.query.BoolQueryBuilder;
44 import org.codelibs.fesen.index.query.QueryBuilders;
45 import org.codelibs.fess.Constants;
46 import org.codelibs.fess.entity.QueryContext;
47 import org.codelibs.fess.entity.SearchRenderData;
48 import org.codelibs.fess.entity.SearchRequestParams;
49 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
50 import org.codelibs.fess.es.client.SearchEngineClient.SearchConditionBuilder;
51 import org.codelibs.fess.es.client.SearchEngineClientException;
52 import org.codelibs.fess.exception.InvalidQueryException;
53 import org.codelibs.fess.mylasta.action.FessUserBean;
54 import org.codelibs.fess.mylasta.direction.FessConfig;
55 import org.codelibs.fess.util.BooleanFunction;
56 import org.codelibs.fess.util.ComponentUtil;
57 import org.codelibs.fess.util.QueryResponseList;
58 import org.dbflute.optional.OptionalEntity;
59 import org.dbflute.optional.OptionalThing;
60 import org.dbflute.util.DfTypeUtil;
61 import org.lastaflute.taglib.function.LaFunctions;
62 import org.lastaflute.web.util.LaRequestUtil;
63
64 public class SearchHelper {
65
66
67
68
69
70 private static final Logger logger = LogManager.getLogger(SearchHelper.class);
71
72
73
74
75
76 public void search(final SearchRequestParams params, final SearchRenderData data, final OptionalThing<FessUserBean> userBean) {
77 final long requestedTime = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
78 final long startTime = System.currentTimeMillis();
79
80 LaRequestUtil.getOptionalRequest().ifPresent(request -> {
81 request.setAttribute(Constants.REQUEST_LANGUAGES, params.getLanguages());
82 request.setAttribute(Constants.REQUEST_QUERIES, params.getQuery());
83 });
84
85 String query = ComponentUtil.getQueryStringBuilder().params(params).sortField(params.getSort()).build();
86 List<Map<String, Object>> documentItems;
87 try {
88 documentItems = searchInternal(query, params, userBean);
89 } catch (final InvalidQueryException e) {
90 if (logger.isDebugEnabled()) {
91 logger.debug("Invalid query: {}", query, e);
92 }
93 query = ComponentUtil.getQueryStringBuilder().params(params).sortField(params.getSort()).escape(true).build();
94 documentItems = searchInternal(query, params, userBean);
95 }
96
97 data.setDocumentItems(documentItems);
98
99
100 final QueryResponseList queryResponseList = (QueryResponseList) documentItems;
101 data.setFacetResponse(queryResponseList.getFacetResponse());
102
103 @SuppressWarnings("unchecked")
104 final Set<String> highlightQueries = (Set<String>) params.getAttribute(Constants.HIGHLIGHT_QUERIES);
105 if (highlightQueries != null) {
106 final StringBuilder buf = new StringBuilder(100);
107 highlightQueries.stream().forEach(q -> {
108 buf.append("&hq=").append(LaFunctions.u(q));
109 });
110 data.setAppendHighlightParams(buf.toString());
111 }
112
113 queryResponseList.setExecTime(System.currentTimeMillis() - startTime);
114 final NumberFormat nf = NumberFormat.getInstance(params.getLocale());
115 nf.setMaximumIntegerDigits(2);
116 nf.setMaximumFractionDigits(2);
117 String execTime;
118 try {
119 execTime = nf.format((double) queryResponseList.getExecTime() / 1000);
120 } catch (final Exception e) {
121 execTime = StringUtil.EMPTY;
122 }
123 data.setExecTime(execTime);
124
125 final String queryId = ComponentUtil.getQueryHelper().generateId();
126
127 data.setPageSize(queryResponseList.getPageSize());
128 data.setCurrentPageNumber(queryResponseList.getCurrentPageNumber());
129 data.setAllRecordCount(queryResponseList.getAllRecordCount());
130 data.setAllRecordCountRelation(queryResponseList.getAllRecordCountRelation());
131 data.setAllPageCount(queryResponseList.getAllPageCount());
132 data.setExistNextPage(queryResponseList.isExistNextPage());
133 data.setExistPrevPage(queryResponseList.isExistPrevPage());
134 data.setCurrentStartRecordNumber(queryResponseList.getCurrentStartRecordNumber());
135 data.setCurrentEndRecordNumber(queryResponseList.getCurrentEndRecordNumber());
136 data.setPageNumberList(queryResponseList.getPageNumberList());
137 data.setPartialResults(queryResponseList.isPartialResults());
138 data.setQueryTime(queryResponseList.getQueryTime());
139 data.setSearchQuery(query);
140 data.setRequestedTime(requestedTime);
141 data.setQueryId(queryId);
142
143 final FessConfig fessConfig = ComponentUtil.getFessConfig();
144
145
146 if (fessConfig.isSearchLog()) {
147 ComponentUtil.getSearchLogHelper().addSearchLog(params, DfTypeUtil.toLocalDateTime(requestedTime), queryId, query,
148 params.getStartPosition(), params.getPageSize(), queryResponseList);
149 }
150
151
152 if (fessConfig.isUserFavorite()) {
153 ComponentUtil.getUserInfoHelper().storeQueryId(queryId, documentItems);
154 }
155
156 }
157
158 protected List<Map<String, Object>> searchInternal(final String query, final SearchRequestParams params,
159 final OptionalThing<FessUserBean> userBean) {
160 final FessConfig fessConfig = ComponentUtil.getFessConfig();
161 final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
162 return ComponentUtil.getSearchEngineClient().search(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> {
163 queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
164 return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(params.getStartPosition())
165 .size(params.getPageSize()).facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo())
166 .highlightInfo(params.getHighlightInfo()).similarDocHash(params.getSimilarDocHash())
167 .responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType())
168 .trackTotalHits(params.getTrackTotalHits()).build();
169 }, (searchRequestBuilder, execTime, searchResponse) -> {
170 searchResponse.ifPresent(r -> {
171 if (r.getTotalShards() != r.getSuccessfulShards() && fessConfig.isQueryTimeoutLogging()) {
172
173 final StringBuilder buf = new StringBuilder(1000);
174 buf.append("[SEARCH TIMEOUT] {\"exec_time\":").append(execTime)
175 .append(",\"request\":").append(searchRequestBuilder.toString())
176 .append(",\"response\":").append(r.toString()).append('}');
177 logger.warn(buf.toString());
178 }
179 });
180 final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
181 queryResponseList.init(searchResponse, params.getStartPosition(), params.getPageSize());
182 return queryResponseList;
183 });
184 }
185
186 public long scrollSearch(final SearchRequestParams params, final BooleanFunction<Map<String, Object>> cursor,
187 final OptionalThing<FessUserBean> userBean) {
188 LaRequestUtil.getOptionalRequest().ifPresent(request -> {
189 request.setAttribute(Constants.REQUEST_LANGUAGES, params.getLanguages());
190 request.setAttribute(Constants.REQUEST_QUERIES, params.getQuery());
191 });
192
193 final int pageSize = params.getPageSize();
194 final String query = ComponentUtil.getQueryStringBuilder().params(params).sortField(params.getSort()).build();
195 final FessConfig fessConfig = ComponentUtil.getFessConfig();
196 return ComponentUtil.getSearchEngineClient().<Map<String, Object>> scrollSearch(fessConfig.getIndexDocumentSearchIndex(),
197 searchRequestBuilder -> {
198 final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
199 queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
200 return SearchConditionBuilder.builder(searchRequestBuilder).scroll().query(query).size(pageSize)
201 .responseFields(queryHelper.getScrollResponseFields()).searchRequestType(params.getType()).build();
202 }, (searchResponse, hit) -> {
203 final Map<String, Object> docMap = new HashMap<>();
204 final Map<String, Object> source = hit.getSourceAsMap();
205 if (source != null) {
206 docMap.putAll(source);
207 }
208 final Map<String, DocumentField> fields = hit.getFields();
209 if (fields != null) {
210 docMap.putAll(fields.entrySet().stream()
211 .collect(Collectors.toMap(Entry::getKey, e -> (Object) e.getValue().getValues())));
212 }
213
214 final ViewHelper viewHelper = ComponentUtil.getViewHelper();
215 if (viewHelper != null && !docMap.isEmpty()) {
216 docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
217 docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
218 docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
219 docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
220 }
221
222 if (!docMap.containsKey(Constants.SCORE)) {
223 docMap.put(Constants.SCORE, hit.getScore());
224 }
225
226 docMap.put(fessConfig.getIndexFieldId(), hit.getId());
227 docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
228 docMap.put(fessConfig.getIndexFieldSeqNo(), hit.getSeqNo());
229 docMap.put(fessConfig.getIndexFieldPrimaryTerm(), hit.getPrimaryTerm());
230 return docMap;
231 }, cursor);
232 }
233
234 public long deleteByQuery(final HttpServletRequest request, final SearchRequestParams params) {
235 final String query = ComponentUtil.getQueryStringBuilder().params(params).build();
236
237 final QueryContext queryContext = ComponentUtil.getQueryHelper().build(params.getType(), query, context -> {
238 context.skipRoleQuery();
239 });
240 return ComponentUtil.getSearchEngineClient().deleteByQuery(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(),
241 queryContext.getQueryBuilder());
242 }
243
244 public String[] getLanguages(final HttpServletRequest request, final SearchRequestParams params) {
245 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
246 if (params.getLanguages() != null) {
247 final Set<String> langSet = new HashSet<>();
248 for (final String lang : params.getLanguages()) {
249 if (StringUtil.isNotBlank(lang) && lang.length() < 1000) {
250 if (Constants.ALL_LANGUAGES.equalsIgnoreCase(lang)) {
251 langSet.add(Constants.ALL_LANGUAGES);
252 } else {
253 final String normalizeLang = systemHelper.normalizeLang(lang);
254 if (normalizeLang != null) {
255 langSet.add(normalizeLang);
256 }
257 }
258 }
259 }
260 if (langSet.size() > 1 && langSet.contains(Constants.ALL_LANGUAGES)) {
261 return new String[] { Constants.ALL_LANGUAGES };
262 }
263 langSet.remove(Constants.ALL_LANGUAGES);
264 return langSet.toArray(new String[langSet.size()]);
265 }
266 if (ComponentUtil.getFessConfig().isBrowserLocaleForSearchUsed()) {
267 final Set<String> langSet = new HashSet<>();
268 final Enumeration<Locale> locales = request.getLocales();
269 if (locales != null) {
270 while (locales.hasMoreElements()) {
271 final Locale locale = locales.nextElement();
272 final String normalizeLang = systemHelper.normalizeLang(locale.toString());
273 if (normalizeLang != null) {
274 langSet.add(normalizeLang);
275 }
276 }
277 if (!langSet.isEmpty()) {
278 return langSet.toArray(new String[langSet.size()]);
279 }
280 }
281 }
282 return StringUtil.EMPTY_STRINGS;
283 }
284
285 public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields,
286 final OptionalThing<FessUserBean> userBean) {
287 final FessConfig fessConfig = ComponentUtil.getFessConfig();
288 return ComponentUtil.getSearchEngineClient().getDocument(fessConfig.getIndexDocumentSearchIndex(), builder -> {
289 final BoolQueryBuilder boolQuery =
290 QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
291 final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON);
292 final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
293 if (!roleSet.isEmpty()) {
294 queryHelper.buildRoleQuery(roleSet, boolQuery);
295 }
296 builder.setQuery(boolQuery);
297 builder.setFetchSource(fields, null);
298 queryHelper.processSearchPreference(builder, userBean, docId);
299 return true;
300 });
301
302 }
303
304 public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields,
305 final OptionalThing<FessUserBean> userBean, final SearchRequestType searchRequestType) {
306 final FessConfig fessConfig = ComponentUtil.getFessConfig();
307 return ComponentUtil.getSearchEngineClient().getDocumentList(fessConfig.getIndexDocumentSearchIndex(), builder -> {
308 final BoolQueryBuilder boolQuery =
309 QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
310 final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
311 if (searchRequestType != SearchRequestType.ADMIN_SEARCH) {
312 final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(searchRequestType);
313 if (!roleSet.isEmpty()) {
314 queryHelper.buildRoleQuery(roleSet, boolQuery);
315 }
316 }
317 builder.setQuery(boolQuery);
318 builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger());
319 builder.setFetchSource(fields, null);
320 queryHelper.processSearchPreference(builder, userBean, String.join(StringUtil.EMPTY, docIds));
321 return true;
322 });
323 }
324
325 public boolean update(final String id, final String field, final Object value) {
326 return ComponentUtil.getSearchEngineClient().update(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(), id, field, value);
327 }
328
329 public boolean update(final String id, final Consumer<UpdateRequestBuilder> builderLambda) {
330 try {
331 final FessConfig fessConfig = ComponentUtil.getFessConfig();
332 final UpdateRequestBuilder builder =
333 ComponentUtil.getSearchEngineClient().prepareUpdate().setIndex(fessConfig.getIndexDocumentUpdateIndex()).setId(id);
334 builderLambda.accept(builder);
335 final UpdateResponse response = builder.execute().actionGet(fessConfig.getIndexIndexTimeout());
336 return response.getResult() == Result.CREATED || response.getResult() == Result.UPDATED;
337 } catch (final FesenException e) {
338 throw new SearchEngineClientException("Failed to update doc " + id, e);
339 }
340 }
341
342 public boolean bulkUpdate(final Consumer<BulkRequestBuilder> consumer) {
343 final BulkRequestBuilder builder = ComponentUtil.getSearchEngineClient().prepareBulk();
344 consumer.accept(builder);
345 try {
346 final BulkResponse response = builder.execute().get();
347 if (response.hasFailures()) {
348 throw new SearchEngineClientException(response.buildFailureMessage());
349 }
350 return true;
351 } catch (InterruptedException | ExecutionException e) {
352 throw new SearchEngineClientException("Failed to update bulk data.", e);
353 }
354 }
355 }