1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20
21 import java.time.LocalDateTime;
22 import java.time.ZoneId;
23 import java.time.format.DateTimeFormatter;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collections;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Set;
32 import java.util.function.Consumer;
33
34 import javax.annotation.PostConstruct;
35
36 import org.apache.logging.log4j.LogManager;
37 import org.apache.logging.log4j.Logger;
38 import org.codelibs.core.lang.StringUtil;
39 import org.codelibs.core.lang.ThreadUtil;
40 import org.codelibs.core.misc.Pair;
41 import org.codelibs.fesen.common.lucene.search.function.CombineFunction;
42 import org.codelibs.fesen.index.query.BoolQueryBuilder;
43 import org.codelibs.fesen.index.query.QueryBuilders;
44 import org.codelibs.fesen.index.query.functionscore.FunctionScoreQueryBuilder;
45 import org.codelibs.fesen.index.query.functionscore.ScoreFunctionBuilders;
46 import org.codelibs.fesen.search.sort.SortBuilders;
47 import org.codelibs.fess.Constants;
48 import org.codelibs.fess.es.client.SearchEngineClient;
49 import org.codelibs.fess.es.config.exbhv.BadWordBhv;
50 import org.codelibs.fess.es.config.exbhv.ElevateWordBhv;
51 import org.codelibs.fess.es.config.exentity.BadWord;
52 import org.codelibs.fess.es.config.exentity.ElevateWord;
53 import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
54 import org.codelibs.fess.es.log.exentity.SearchLog;
55 import org.codelibs.fess.mylasta.direction.FessConfig;
56 import org.codelibs.fess.suggest.Suggester;
57 import org.codelibs.fess.suggest.constants.FieldNames;
58 import org.codelibs.fess.suggest.entity.SuggestItem;
59 import org.codelibs.fess.suggest.exception.SuggestSettingsException;
60 import org.codelibs.fess.suggest.index.SuggestDeleteResponse;
61 import org.codelibs.fess.suggest.index.contents.document.ESSourceReader;
62 import org.codelibs.fess.suggest.settings.SuggestSettings;
63 import org.codelibs.fess.suggest.settings.SuggestSettingsBuilder;
64 import org.codelibs.fess.suggest.util.SuggestUtil;
65 import org.codelibs.fess.util.ComponentUtil;
66
67 public class SuggestHelper {
68 private static final Logger logger = LogManager.getLogger(SuggestHelper.class);
69
70 protected static final String TEXT_SEP = " ";
71
72 protected Suggester suggester;
73
74 protected FessConfig fessConfig;
75
76 protected final Set<String> contentFieldNameSet = new HashSet<>();
77
78 protected final Set<String> tagFieldNameSet = new HashSet<>();
79
80 protected final Set<String> roleFieldNameSet = new HashSet<>();
81
82 protected List<String> contentFieldList;
83
84 protected PopularWordHelper popularWordHelper = null;
85
86 public long searchStoreIntervalMinute = 1;
87
88 @PostConstruct
89 public void init() {
90 if (logger.isDebugEnabled()) {
91 logger.debug("Initialize {}", this.getClass().getSimpleName());
92 }
93 fessConfig = ComponentUtil.getFessConfig();
94 split(fessConfig.getSuggestFieldContents(), ",")
95 .of(stream -> stream.filter(StringUtil::isNotBlank).forEach(contentFieldNameSet::add));
96 split(fessConfig.getSuggestFieldTags(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(tagFieldNameSet::add));
97 split(fessConfig.getSuggestFieldRoles(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(roleFieldNameSet::add));
98 contentFieldList = Arrays.asList(stream(fessConfig.getSuggestFieldContents()).get(stream -> stream.toArray(n -> new String[n])));
99
100 final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
101 searchEngineClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute()
102 .actionGet(fessConfig.getIndexHealthTimeout());
103
104 final SuggestSettingsBuilder settingsBuilder = SuggestSettings.builder();
105 settingsBuilder.addInitialSettings("elasticsearch.type", fessConfig.getFesenType());
106 settingsBuilder.bulkTimeout(fessConfig.getIndexBulkTimeout());
107 settingsBuilder.clusterTimeout(fessConfig.getIndexHealthTimeout());
108 settingsBuilder.indexTimeout(fessConfig.getIndexIndexTimeout());
109 settingsBuilder.indicesTimeout(fessConfig.getIndexIndicesTimeout());
110 settingsBuilder.searchTimeout(fessConfig.getIndexSearchTimeout());
111 suggester = Suggester.builder().settings(settingsBuilder).build(searchEngineClient, fessConfig.getIndexDocumentSuggestIndex());
112 suggester.settings().array().delete(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS);
113 split(fessConfig.getSuggestFieldIndexContents(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(field -> {
114 try {
115 suggester.settings().array().add(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS, field);
116 } catch (final SuggestSettingsException e) {
117 logger.warn("Failed to add {}", field, e);
118 }
119 }));
120 suggester.createIndexIfNothing();
121
122 if (ComponentUtil.hasPopularWordHelper()) {
123 popularWordHelper = ComponentUtil.getPopularWordHelper();
124 }
125 }
126
127 public Suggester suggester() {
128 return suggester;
129 }
130
131 public void storeSearchLog() {
132 final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
133
134 searchLogBhv.selectBulk(cb -> {
135 final String from = LocalDateTime.now().minusDays(fessConfig.getPurgeSuggestSearchLogDay()).format(DateTimeFormatter.ISO_DATE);
136 cb.query().addQuery(QueryBuilders.rangeQuery("requestedAt").gte(from));
137 cb.query().addOrderBy_RequestedAt_Asc();
138 }, this::indexFromSearchLog);
139 }
140
141 public void indexFromSearchLog(final List<SearchLog> searchLogList) {
142 final Map<String, LocalDateTime> duplicateSessionMap = new HashMap<>();
143 searchLogList.stream().forEach(searchLog -> {
144 if (searchLog.getHitCount() == null
145 || searchLog.getHitCount().longValue() < fessConfig.getSuggestMinHitCountAsInteger().longValue()) {
146 return;
147 }
148
149 final String sessionId;
150 if (searchLog.getUserSessionId() != null) {
151 sessionId = searchLog.getUserSessionId();
152 } else if (Constants.SEARCH_LOG_ACCESS_TYPE_WEB.equals(searchLog.getAccessType())) {
153 sessionId = searchLog.getClientIp();
154 } else {
155 sessionId = searchLog.getClientIp() + '_' + searchLog.getSearchWord();
156 }
157
158 final LocalDateTime requestedAt = searchLog.getRequestedAt();
159 if ((sessionId == null) || (duplicateSessionMap.containsKey(sessionId)
160 && duplicateSessionMap.get(sessionId).plusMinutes(searchStoreIntervalMinute).isAfter(requestedAt))) {
161 return;
162 }
163
164 final StringBuilder sb = new StringBuilder(100);
165 final List<String> fields = new ArrayList<>();
166 final List<String> tags = new ArrayList<>();
167 final List<String> roles = new ArrayList<>();
168
169 for (final Pair<String, String> searchFieldLog : searchLog.getSearchFieldLogList()) {
170 final String name = searchFieldLog.getFirst();
171 if (contentFieldNameSet.contains(name)) {
172 if (sb.length() > 0) {
173 sb.append(TEXT_SEP);
174 }
175 sb.append(searchFieldLog.getSecond());
176 fields.add(name);
177 } else if (tagFieldNameSet.contains(name)) {
178 tags.add(searchFieldLog.getSecond());
179 } else if (roleFieldNameSet.contains(name)) {
180 roles.add(searchFieldLog.getSecond());
181 }
182 }
183
184 final String virtualHost = searchLog.getVirtualHost();
185 if (virtualHost != null) {
186 tags.add(virtualHost);
187 }
188
189 if (sb.length() > 0) {
190 final String[] langs = searchLog.getLanguages() == null ? new String[] {} : searchLog.getLanguages().split(",");
191 stream(searchLog.getRoles()).of(stream -> stream.forEach(role -> roles.add(role)));
192 if (fessConfig.isValidSearchLogPermissions(roles.toArray(new String[roles.size()]))) {
193 suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]),
194 tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1, langs);
195 duplicateSessionMap.put(sessionId, requestedAt);
196 }
197 }
198 });
199 refresh();
200 }
201
202 public void indexFromDocuments(final Consumer<Boolean> success, final Consumer<Throwable> error) {
203 final FessConfig fessConfig = ComponentUtil.getFessConfig();
204 final long interval = fessConfig.getSuggestUpdateRequestIntervalAsInteger().longValue();
205 final int docPerReq = fessConfig.getSuggestUpdateDocPerRequestAsInteger();
206 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
207 suggester.indexer().indexFromDocument(() -> {
208 final ESSourceReader reader = new ESSourceReader(ComponentUtil.getSearchEngineClient(), suggester.settings(),
209 fessConfig.getIndexDocumentSearchIndex(), "_doc");
210 reader.setScrollSize(fessConfig.getSuggestSourceReaderScrollSizeAsInteger());
211 reader.setLimitDocNumPercentage(fessConfig.getSuggestUpdateContentsLimitNumPercentage());
212 reader.setLimitNumber(fessConfig.getSuggestUpdateContentsLimitNumAsInteger());
213 reader.setLimitOfDocumentSize(fessConfig.getSuggestUpdateContentsLimitDocSizeAsInteger());
214
215 final List<FunctionScoreQueryBuilder.FilterFunctionBuilder> flist = new ArrayList<>();
216 flist.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(
217 ScoreFunctionBuilders.randomFunction().seed(System.currentTimeMillis()).setField(fessConfig.getIndexFieldDocId())));
218 reader.setQuery(QueryBuilders
219 .functionScoreQuery(QueryBuilders.matchAllQuery(),
220 flist.toArray(new FunctionScoreQueryBuilder.FilterFunctionBuilder[flist.size()]))
221 .boostMode(CombineFunction.MULTIPLY));
222 reader.addSort(SortBuilders.fieldSort(fessConfig.getIndexFieldClickCount()));
223 reader.addSort(SortBuilders.scoreSort());
224 return reader;
225 }, docPerReq, () -> {
226 systemHelper.calibrateCpuLoad();
227 ThreadUtil.sleep(interval);
228 }).then(response -> {
229 refresh();
230 success.accept(true);
231 }).error(t -> error.accept(t));
232 }
233
234 public void purgeDocumentSuggest(final LocalDateTime time) {
235 final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
236 boolQueryBuilder
237 .must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()));
238
239 boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()));
240 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
241 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
242
243 SuggestUtil.deleteByQuery(ComponentUtil.getSearchEngineClient(), suggester.settings(), suggester.getIndex(), boolQueryBuilder);
244 }
245
246 public void purgeSearchlogSuggest(final LocalDateTime time) {
247 final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
248 boolQueryBuilder
249 .must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()));
250
251 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()));
252 boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
253 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
254
255 SuggestUtil.deleteByQuery(ComponentUtil.getSearchEngineClient(), suggester.settings(), suggester.getIndex(), boolQueryBuilder);
256 }
257
258 public long getAllWordsNum() {
259 return suggester.getAllWordsNum();
260 }
261
262 public long getDocumentWordsNum() {
263 return suggester.getDocumentWordsNum();
264 }
265
266 public long getQueryWordsNum() {
267 return suggester.getQueryWordsNum();
268 }
269
270 public boolean deleteAllWords() {
271 final SuggestDeleteResponse response = suggester.indexer().deleteAll();
272 if (response.hasError()) {
273 logger.warn("Failed to delete all words.", response.getErrors().get(0));
274 return false;
275 }
276 refresh();
277 return true;
278 }
279
280 public boolean deleteDocumentWords() {
281 final SuggestDeleteResponse response = suggester.indexer().deleteDocumentWords();
282 if (response.hasError()) {
283 logger.warn("Failed to delete document words.", response.getErrors().get(0));
284 return false;
285 }
286 refresh();
287 return true;
288 }
289
290 public boolean deleteQueryWords() {
291 final SuggestDeleteResponse response = suggester.indexer().deleteQueryWords();
292 if (response.hasError()) {
293 logger.warn("Failed to delete query words.", response.getErrors().get(0));
294 return false;
295 }
296 refresh();
297 return true;
298 }
299
300 public void storeAllElevateWords(final boolean apply) {
301 deleteAllElevateWord(apply);
302
303 final List<ElevateWord> list = ComponentUtil.getComponent(ElevateWordBhv.class).selectList(cb -> {
304 cb.query().matchAll();
305 cb.fetchFirst(ComponentUtil.getFessConfig().getPageElevateWordMaxFetchSizeAsInteger());
306 });
307
308 for (final ElevateWord elevateWord : list) {
309 addElevateWord(elevateWord.getSuggestWord(), elevateWord.getReading(), elevateWord.getLabelTypeValues(),
310 elevateWord.getPermissions(), elevateWord.getBoost(), apply);
311 }
312 refresh();
313 }
314
315 public void deleteAllElevateWord(final boolean apply) {
316 final List<ElevateWord> list = ComponentUtil.getComponent(ElevateWordBhv.class).selectList(cb -> {
317 cb.query().matchAll();
318 cb.fetchFirst(ComponentUtil.getFessConfig().getPageElevateWordMaxFetchSizeAsInteger());
319 });
320
321 for (final ElevateWord elevateWord : list) {
322 suggester.indexer().deleteElevateWord(elevateWord.getSuggestWord(), apply);
323 }
324 refresh();
325 }
326
327 public void deleteElevateWord(final String word, final boolean apply) {
328 suggester.indexer().deleteElevateWord(word, apply);
329 refresh();
330 }
331
332 public void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, final Float boost,
333 final boolean apply) {
334 final String[] readings;
335 if (StringUtil.isBlank(reading)) {
336 readings = word.replace(" ", TEXT_SEP).replaceAll(TEXT_SEP + "+", TEXT_SEP).split(TEXT_SEP);
337 } else {
338 readings = reading.replace(" ", TEXT_SEP).replaceAll(TEXT_SEP + "+", TEXT_SEP).split(TEXT_SEP);
339 }
340
341 final List<String> labelList = new ArrayList<>();
342 if (tags != null) {
343 Collections.addAll(labelList, tags);
344 }
345 final List<String> roleList = new ArrayList<>();
346 if (permissions != null) {
347 Collections.addAll(roleList, permissions);
348 }
349
350 suggester.indexer().addElevateWord(new org.codelibs.fess.suggest.entity.ElevateWord(word, boost, Arrays.asList(readings),
351 contentFieldList, labelList, roleList), apply);
352
353 refresh();
354 }
355
356 protected void deleteAllBadWords() {
357 suggester.settings().badword().deleteAll();
358 }
359
360 public void storeAllBadWords(final boolean apply) {
361 deleteAllBadWords();
362 final List<BadWord> list = ComponentUtil.getComponent(BadWordBhv.class).selectList(cb -> {
363 cb.query().matchAll();
364 cb.fetchFirst(ComponentUtil.getFessConfig().getPageBadWordMaxFetchSizeAsInteger());
365 });
366 for (final BadWord badWord : list) {
367 final String word = badWord.getSuggestWord();
368 suggester.indexer().addBadWord(word, apply);
369 }
370 refresh();
371 }
372
373 public void addBadWord(final String badWord, final boolean apply) {
374 suggester.indexer().addBadWord(badWord, apply);
375 refresh();
376 }
377
378 public void deleteBadWord(final String badWord) {
379 suggester.indexer().deleteBadWord(badWord);
380 refresh();
381 }
382
383 public synchronized void refresh() {
384 suggester.refresh();
385 if (popularWordHelper != null) {
386 popularWordHelper.clearCache();
387 }
388 }
389 }