View Javadoc
1   /*
2    * Copyright 2012-2020 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.helper;
17  
18  import java.text.NumberFormat;
19  import java.util.Enumeration;
20  import java.util.HashMap;
21  import java.util.HashSet;
22  import java.util.List;
23  import java.util.Locale;
24  import java.util.Map;
25  import java.util.Map.Entry;
26  import java.util.Set;
27  import java.util.concurrent.ExecutionException;
28  import java.util.function.Consumer;
29  import java.util.stream.Collectors;
30  
31  import javax.servlet.http.HttpServletRequest;
32  
33  import org.apache.logging.log4j.LogManager;
34  import org.apache.logging.log4j.Logger;
35  import org.codelibs.core.lang.StringUtil;
36  import org.codelibs.fess.Constants;
37  import org.codelibs.fess.entity.QueryContext;
38  import org.codelibs.fess.entity.SearchRenderData;
39  import org.codelibs.fess.entity.SearchRequestParams;
40  import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
41  import org.codelibs.fess.es.client.FessEsClient.SearchConditionBuilder;
42  import org.codelibs.fess.es.client.FessEsClientException;
43  import org.codelibs.fess.mylasta.action.FessUserBean;
44  import org.codelibs.fess.mylasta.direction.FessConfig;
45  import org.codelibs.fess.util.BooleanFunction;
46  import org.codelibs.fess.util.ComponentUtil;
47  import org.codelibs.fess.util.QueryResponseList;
48  import org.dbflute.optional.OptionalEntity;
49  import org.dbflute.optional.OptionalThing;
50  import org.dbflute.util.DfTypeUtil;
51  import org.elasticsearch.ElasticsearchException;
52  import org.elasticsearch.action.DocWriteResponse.Result;
53  import org.elasticsearch.action.bulk.BulkRequestBuilder;
54  import org.elasticsearch.action.bulk.BulkResponse;
55  import org.elasticsearch.action.update.UpdateRequestBuilder;
56  import org.elasticsearch.action.update.UpdateResponse;
57  import org.elasticsearch.common.document.DocumentField;
58  import org.elasticsearch.index.query.BoolQueryBuilder;
59  import org.elasticsearch.index.query.QueryBuilders;
60  import org.lastaflute.taglib.function.LaFunctions;
61  import org.lastaflute.web.util.LaRequestUtil;
62  
63  public class SearchHelper {
64  
65      // ===================================================================================
66      //                                                                            Constant
67      //
68  
69      private static final Logger logger = LogManager.getLogger(SearchHelper.class);
70  
71      // ===================================================================================
72      //                                                                              Method
73      //                                                                      ==============
74  
75      public void search(final SearchRequestParams params, final SearchRenderData data, final OptionalThing<FessUserBean> userBean) {
76          final long requestedTime = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
77          final long startTime = System.currentTimeMillis();
78  
79          LaRequestUtil.getOptionalRequest().ifPresent(request -> {
80              request.setAttribute(Constants.REQUEST_LANGUAGES, params.getLanguages());
81              request.setAttribute(Constants.REQUEST_QUERIES, params.getQuery());
82          });
83  
84          final int pageStart = params.getStartPosition();
85          final int pageSize = params.getPageSize();
86          final String sortField = params.getSort();
87          final String query;
88          if (StringUtil.isBlank(sortField)) {
89              query = ComponentUtil.getQueryStringBuilder().params(params).build();
90          } else {
91              query = ComponentUtil.getQueryStringBuilder().params(params).build() + " sort:" + sortField;
92          }
93          final FessConfig fessConfig = ComponentUtil.getFessConfig();
94          final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
95          final List<Map<String, Object>> documentItems =
96                  ComponentUtil.getFessEsClient().search(
97                          fessConfig.getIndexDocumentSearchIndex(),
98                          searchRequestBuilder -> {
99                              queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
100                             return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(pageStart).size(pageSize)
101                                     .facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo()).highlightInfo(params.getHighlightInfo())
102                                     .similarDocHash(params.getSimilarDocHash()).responseFields(queryHelper.getResponseFields())
103                                     .searchRequestType(params.getType()).trackTotalHits(params.getTrackTotalHits()).build();
104                         }, (searchRequestBuilder, execTime, searchResponse) -> {
105                             searchResponse.ifPresent(r -> {
106                                 if (r.getTotalShards() != r.getSuccessfulShards() && fessConfig.isQueryTimeoutLogging()) {
107                                     // partial results
108                                     final StringBuilder buf = new StringBuilder(1000);
109                                     buf.append("[SEARCH TIMEOUT] {\"exec_time\":").append(execTime)//
110                                             .append(",\"request\":").append(searchRequestBuilder.toString())//
111                                             .append(",\"response\":").append(r.toString()).append('}');
112                                     logger.warn(buf.toString());
113                                 }
114                             });
115                             final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
116                             queryResponseList.init(searchResponse, pageStart, pageSize);
117                             return queryResponseList;
118                         });
119         data.setDocumentItems(documentItems);
120 
121         // search
122         final QueryResponseListbs/fess/util/QueryResponseList.html#QueryResponseList">QueryResponseList queryResponseList = (QueryResponseList) documentItems;
123         data.setFacetResponse(queryResponseList.getFacetResponse());
124 
125         @SuppressWarnings("unchecked")
126         final Set<String> highlightQueries = (Set<String>) params.getAttribute(Constants.HIGHLIGHT_QUERIES);
127         if (highlightQueries != null) {
128             final StringBuilder buf = new StringBuilder(100);
129             highlightQueries.stream().forEach(q -> {
130                 buf.append("&hq=").append(LaFunctions.u(q));
131             });
132             data.setAppendHighlightParams(buf.toString());
133         }
134 
135         queryResponseList.setExecTime(System.currentTimeMillis() - startTime);
136         final NumberFormat nf = NumberFormat.getInstance(params.getLocale());
137         nf.setMaximumIntegerDigits(2);
138         nf.setMaximumFractionDigits(2);
139         String execTime;
140         try {
141             execTime = nf.format((double) queryResponseList.getExecTime() / 1000);
142         } catch (final Exception e) {
143             execTime = StringUtil.EMPTY;
144         }
145         data.setExecTime(execTime);
146 
147         final String queryId = queryHelper.generateId();
148 
149         data.setPageSize(queryResponseList.getPageSize());
150         data.setCurrentPageNumber(queryResponseList.getCurrentPageNumber());
151         data.setAllRecordCount(queryResponseList.getAllRecordCount());
152         data.setAllRecordCountRelation(queryResponseList.getAllRecordCountRelation());
153         data.setAllPageCount(queryResponseList.getAllPageCount());
154         data.setExistNextPage(queryResponseList.isExistNextPage());
155         data.setExistPrevPage(queryResponseList.isExistPrevPage());
156         data.setCurrentStartRecordNumber(queryResponseList.getCurrentStartRecordNumber());
157         data.setCurrentEndRecordNumber(queryResponseList.getCurrentEndRecordNumber());
158         data.setPageNumberList(queryResponseList.getPageNumberList());
159         data.setPartialResults(queryResponseList.isPartialResults());
160         data.setQueryTime(queryResponseList.getQueryTime());
161         data.setSearchQuery(query);
162         data.setRequestedTime(requestedTime);
163         data.setQueryId(queryId);
164 
165         // search log
166         if (fessConfig.isSearchLog()) {
167             ComponentUtil.getSearchLogHelper().addSearchLog(params, DfTypeUtil.toLocalDateTime(requestedTime), queryId, query, pageStart,
168                     pageSize, queryResponseList);
169         }
170 
171         // favorite
172         if (fessConfig.isUserFavorite()) {
173             ComponentUtil.getUserInfoHelper().storeQueryId(queryId, documentItems);
174         }
175 
176     }
177 
178     public long scrollSearch(final SearchRequestParams params, final BooleanFunction<Map<String, Object>> cursor,
179             final OptionalThing<FessUserBean> userBean) {
180         LaRequestUtil.getOptionalRequest().ifPresent(request -> {
181             request.setAttribute(Constants.REQUEST_LANGUAGES, params.getLanguages());
182             request.setAttribute(Constants.REQUEST_QUERIES, params.getQuery());
183         });
184 
185         final int pageSize = params.getPageSize();
186         final String sortField = params.getSort();
187         final String query;
188         if (StringUtil.isBlank(sortField)) {
189             query = ComponentUtil.getQueryStringBuilder().params(params).build();
190         } else {
191             query = ComponentUtil.getQueryStringBuilder().params(params).build() + " sort:" + sortField;
192         }
193         final FessConfig fessConfig = ComponentUtil.getFessConfig();
194         return ComponentUtil.getFessEsClient().<Map<String, Object>> scrollSearch(
195                 fessConfig.getIndexDocumentSearchIndex(),
196                 searchRequestBuilder -> {
197                     final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
198                     queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
199                     return SearchConditionBuilder.builder(searchRequestBuilder).scroll().query(query).size(pageSize)
200                             .responseFields(queryHelper.getScrollResponseFields()).searchRequestType(params.getType()).build();
201                 },
202                 (searchResponse, hit) -> {
203                     final Map<String, Object> docMap = new HashMap<>();
204                     final Map<String, Object> source = hit.getSourceAsMap();
205                     if (source != null) {
206                         docMap.putAll(source);
207                     }
208                     final Map<String, DocumentField> fields = hit.getFields();
209                     if (fields != null) {
210                         docMap.putAll(fields.entrySet().stream()
211                                 .collect(Collectors.toMap(Entry::getKey, e -> (Object) e.getValue().getValues())));
212                     }
213 
214                     final ViewHelper viewHelper = ComponentUtil.getViewHelper();
215                     if (viewHelper != null && !docMap.isEmpty()) {
216                         docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
217                         docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
218                         docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
219                         docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
220                     }
221 
222                     if (!docMap.containsKey(Constants.SCORE)) {
223                         docMap.put(Constants.SCORE, hit.getScore());
224                     }
225 
226                     docMap.put(fessConfig.getIndexFieldId(), hit.getId());
227                     docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
228                     docMap.put(fessConfig.getIndexFieldSeqNo(), hit.getSeqNo());
229                     docMap.put(fessConfig.getIndexFieldPrimaryTerm(), hit.getPrimaryTerm());
230                     return docMap;
231                 }, cursor);
232     }
233 
234     public long deleteByQuery(final HttpServletRequest request, final SearchRequestParams params) {
235         final String query = ComponentUtil.getQueryStringBuilder().params(params).build();
236 
237         final QueryContext queryContext = ComponentUtil.getQueryHelper().build(params.getType(), query, context -> {
238             context.skipRoleQuery();
239         });
240         return ComponentUtil.getFessEsClient().deleteByQuery(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(),
241                 queryContext.getQueryBuilder());
242     }
243 
244     public String[] getLanguages(final HttpServletRequest request, final SearchRequestParams params) {
245         final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
246         if (params.getLanguages() != null) {
247             final Set<String> langSet = new HashSet<>();
248             for (final String lang : params.getLanguages()) {
249                 if (StringUtil.isNotBlank(lang) && lang.length() < 1000) {
250                     if (Constants.ALL_LANGUAGES.equalsIgnoreCase(lang)) {
251                         langSet.add(Constants.ALL_LANGUAGES);
252                     } else {
253                         final String normalizeLang = systemHelper.normalizeLang(lang);
254                         if (normalizeLang != null) {
255                             langSet.add(normalizeLang);
256                         }
257                     }
258                 }
259             }
260             if (langSet.size() > 1 && langSet.contains(Constants.ALL_LANGUAGES)) {
261                 return new String[] { Constants.ALL_LANGUAGES };
262             } else {
263                 langSet.remove(Constants.ALL_LANGUAGES);
264             }
265             return langSet.toArray(new String[langSet.size()]);
266         } else if (ComponentUtil.getFessConfig().isBrowserLocaleForSearchUsed()) {
267             final Set<String> langSet = new HashSet<>();
268             final Enumeration<Locale> locales = request.getLocales();
269             if (locales != null) {
270                 while (locales.hasMoreElements()) {
271                     final Locale locale = locales.nextElement();
272                     final String normalizeLang = systemHelper.normalizeLang(locale.toString());
273                     if (normalizeLang != null) {
274                         langSet.add(normalizeLang);
275                     }
276                 }
277                 if (!langSet.isEmpty()) {
278                     return langSet.toArray(new String[langSet.size()]);
279                 }
280             }
281         }
282         return StringUtil.EMPTY_STRINGS;
283     }
284 
285     public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields,
286             final OptionalThing<FessUserBean> userBean) {
287         final FessConfig fessConfig = ComponentUtil.getFessConfig();
288         return ComponentUtil.getFessEsClient().getDocument(
289                 fessConfig.getIndexDocumentSearchIndex(),
290                 builder -> {
291                     final BoolQueryBuilder boolQuery =
292                             QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
293                     final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON); // TODO SearchRequestType?
294                     final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
295                     if (!roleSet.isEmpty()) {
296                         queryHelper.buildRoleQuery(roleSet, boolQuery);
297                     }
298                     builder.setQuery(boolQuery);
299                     builder.setFetchSource(fields, null);
300                     queryHelper.processSearchPreference(builder, userBean, docId);
301                     return true;
302                 });
303 
304     }
305 
306     public List<Map<String, Object>> getDocumentListByDocIds(final String[] docIds, final String[] fields,
307             final OptionalThing<FessUserBean> userBean, final SearchRequestType searchRequestType) {
308         final FessConfig fessConfig = ComponentUtil.getFessConfig();
309         return ComponentUtil.getFessEsClient().getDocumentList(
310                 fessConfig.getIndexDocumentSearchIndex(),
311                 builder -> {
312                     final BoolQueryBuilder boolQuery =
313                             QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIds));
314                     final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
315                     if (searchRequestType != SearchRequestType.ADMIN_SEARCH) {
316                         final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(searchRequestType);
317                         if (!roleSet.isEmpty()) {
318                             queryHelper.buildRoleQuery(roleSet, boolQuery);
319                         }
320                     }
321                     builder.setQuery(boolQuery);
322                     builder.setSize(fessConfig.getPagingSearchPageMaxSizeAsInteger());
323                     builder.setFetchSource(fields, null);
324                     queryHelper.processSearchPreference(builder, userBean, String.join(StringUtil.EMPTY, docIds));
325                     return true;
326                 });
327     }
328 
329     public boolean update(final String id, final String field, final Object value) {
330         return ComponentUtil.getFessEsClient().update(ComponentUtil.getFessConfig().getIndexDocumentUpdateIndex(), id, field, value);
331     }
332 
333     public boolean update(final String id, final Consumer<UpdateRequestBuilder> builderLambda) {
334         try {
335             final FessConfig fessConfig = ComponentUtil.getFessConfig();
336             final UpdateRequestBuilder builder =
337                     ComponentUtil.getFessEsClient().prepareUpdate().setIndex(fessConfig.getIndexDocumentUpdateIndex()).setId(id);
338             builderLambda.accept(builder);
339             final UpdateResponse response = builder.execute().actionGet(fessConfig.getIndexIndexTimeout());
340             return response.getResult() == Result.CREATED || response.getResult() == Result.UPDATED;
341         } catch (final ElasticsearchException e) {
342             throw new FessEsClientException("Failed to update doc  " + id, e);
343         }
344     }
345 
346     public boolean bulkUpdate(final Consumer<BulkRequestBuilder> consumer) {
347         final BulkRequestBuilder builder = ComponentUtil.getFessEsClient().prepareBulk();
348         consumer.accept(builder);
349         try {
350             final BulkResponse response = builder.execute().get();
351             if (response.hasFailures()) {
352                 throw new FessEsClientException(response.buildFailureMessage());
353             } else {
354                 return true;
355             }
356         } catch (InterruptedException | ExecutionException e) {
357             throw new FessEsClientException("Failed to update bulk data.", e);
358         }
359     }
360 }