1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20
21 import java.time.LocalDateTime;
22 import java.time.ZoneId;
23 import java.time.format.DateTimeFormatter;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Set;
29 import java.util.function.Consumer;
30
31 import javax.annotation.PostConstruct;
32 import javax.annotation.Resource;
33
34 import org.codelibs.core.lang.StringUtil;
35 import org.codelibs.core.misc.Pair;
36 import org.codelibs.fess.es.client.FessEsClient;
37 import org.codelibs.fess.es.config.exbhv.BadWordBhv;
38 import org.codelibs.fess.es.config.exbhv.ElevateWordBhv;
39 import org.codelibs.fess.es.config.exentity.BadWord;
40 import org.codelibs.fess.es.config.exentity.ElevateWord;
41 import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
42 import org.codelibs.fess.es.log.exentity.SearchLog;
43 import org.codelibs.fess.mylasta.direction.FessConfig;
44 import org.codelibs.fess.suggest.Suggester;
45 import org.codelibs.fess.suggest.constants.FieldNames;
46 import org.codelibs.fess.suggest.entity.SuggestItem;
47 import org.codelibs.fess.suggest.index.SuggestDeleteResponse;
48 import org.codelibs.fess.suggest.index.contents.document.ESSourceReader;
49 import org.codelibs.fess.suggest.settings.SuggestSettings;
50 import org.codelibs.fess.suggest.util.SuggestUtil;
51 import org.codelibs.fess.util.ComponentUtil;
52 import org.elasticsearch.common.lucene.search.function.CombineFunction;
53 import org.elasticsearch.index.query.BoolQueryBuilder;
54 import org.elasticsearch.index.query.QueryBuilders;
55 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
56 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
57 import org.elasticsearch.search.sort.SortBuilders;
58 import org.slf4j.Logger;
59 import org.slf4j.LoggerFactory;
60
61 public class SuggestHelper {
62 private static final Logger logger = LoggerFactory.getLogger(SuggestHelper.class);
63
64 private static final String TEXT_SEP = " ";
65
66 @Resource
67 protected ElevateWordBhv elevateWordBhv;
68
69 @Resource
70 protected BadWordBhv badWordBhv;
71
72 @Resource
73 protected FessEsClient fessEsClient;
74
75 protected Suggester suggester;
76
77 protected FessConfig fessConfig;
78
79 protected final Set<String> contentFieldNameSet = new HashSet<>();
80
81 protected final Set<String> tagFieldNameSet = new HashSet<>();
82
83 protected final Set<String> roleFieldNameSet = new HashSet<>();
84
85 protected List<String> contentFieldList;
86
87 protected PopularWordHelper popularWordHelper = null;
88
89 @PostConstruct
90 public void init() {
91 fessConfig = ComponentUtil.getFessConfig();
92 split(fessConfig.getSuggestFieldContents(), ",").of(
93 stream -> stream.filter(StringUtil::isNotBlank).forEach(contentFieldNameSet::add));
94 split(fessConfig.getSuggestFieldTags(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(tagFieldNameSet::add));
95 split(fessConfig.getSuggestFieldRoles(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(roleFieldNameSet::add));
96 contentFieldList = Arrays.asList(stream(fessConfig.getSuggestFieldContents()).get(stream -> stream.toArray(n -> new String[n])));
97
98 fessEsClient.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(fessConfig.getIndexHealthTimeout());
99
100 suggester = Suggester.builder().build(fessEsClient, fessConfig.getIndexDocumentSuggestIndex());
101 suggester.settings().array().delete(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS);
102 split(fessConfig.getSuggestFieldIndexContents(), ",").of(
103 stream -> stream.filter(StringUtil::isNotBlank).forEach(
104 field -> suggester.settings().array().add(SuggestSettings.DefaultKeys.SUPPORTED_FIELDS, field)));
105 suggester.createIndexIfNothing();
106
107 final Set<String> undefinedAnalyzer = suggester.settings().analyzer().checkAnalyzer();
108 if (undefinedAnalyzer.size() > 0) {
109 logger.warn("Undefined lang analyzer. " + undefinedAnalyzer.toString());
110 }
111
112 if (ComponentUtil.hasPopularWordHelper()) {
113 popularWordHelper = ComponentUtil.getPopularWordHelper();
114 }
115 }
116
117 public Suggester suggester() {
118 return suggester;
119 }
120
121 public void storeSearchLog() {
122 final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
123
124 searchLogBhv.selectBulk((cb) -> {
125 final String from = LocalDateTime.now().minusDays(fessConfig.getPurgeSuggestSearchLogDay()).format(DateTimeFormatter.ISO_DATE);
126 cb.query().addQuery(QueryBuilders.rangeQuery("requestedAt").gte(from));
127 cb.query().addOrderBy_RequestedAt_Asc();
128 }, (searchLogsList) -> indexFromSearchLog(searchLogsList));
129 }
130
131 public void indexFromSearchLog(final List<SearchLog> searchLogList) {
132 final Set<String> sessionIdSet = new HashSet<>();
133 searchLogList.stream().forEach(
134 searchLog -> {
135 if (searchLog.getHitCount() == null
136 || searchLog.getHitCount().longValue() < fessConfig.getSuggestMinHitCountAsInteger().longValue()) {
137 return;
138 }
139
140 final String sessionId = searchLog.getUserSessionId();
141 if (sessionId == null || sessionIdSet.contains(sessionId)) {
142 return;
143 }
144
145 final StringBuilder sb = new StringBuilder(100);
146 final List<String> fields = new ArrayList<>();
147 final List<String> tags = new ArrayList<>();
148 final List<String> roles = new ArrayList<>();
149
150 for (final Pair<String, String> searchFieldLog : searchLog.getSearchFieldLogList()) {
151 final String name = searchFieldLog.getFirst();
152 if (contentFieldNameSet.contains(name)) {
153 if (sb.length() > 0) {
154 sb.append(TEXT_SEP);
155 }
156 sb.append(searchFieldLog.getSecond());
157 fields.add(name);
158 } else if (tagFieldNameSet.contains(name)) {
159 tags.add(searchFieldLog.getSecond());
160 } else if (roleFieldNameSet.contains(name)) {
161 roles.add(searchFieldLog.getSecond());
162 }
163 }
164
165 final String virtualHost = searchLog.getVirtualHost();
166 if (virtualHost != null) {
167 tags.add(virtualHost);
168 }
169
170 if (sb.length() > 0) {
171 final String[] langs = searchLog.getLanguages() == null ? new String[] {} : searchLog.getLanguages().split(",");
172 stream(searchLog.getRoles()).of(stream -> stream.forEach(role -> roles.add(role)));
173 if (fessConfig.isValidSearchLogPermissions(roles.toArray(new String[roles.size()]))) {
174 suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]),
175 tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1, langs);
176 sessionIdSet.add(sessionId);
177 }
178 }
179 });
180 refresh();
181 }
182
183 public void indexFromDocuments(final Consumer<Boolean> success, final Consumer<Throwable> error) {
184 final FessConfig fessConfig = ComponentUtil.getFessConfig();
185 suggester
186 .indexer()
187 .indexFromDocument(
188 () -> {
189 final ESSourceReader reader =
190 new ESSourceReader(fessEsClient, suggester.settings(), fessConfig.getIndexDocumentSearchIndex(),
191 fessConfig.getIndexDocumentType());
192 reader.setScrollSize(fessConfig.getSuggestSourceReaderScrollSizeAsInteger());
193 reader.setLimitDocNumPercentage(fessConfig.getSuggestUpdateContentsLimitNumPercentage());
194 reader.setLimitNumber(fessConfig.getSuggestUpdateContentsLimitNumAsInteger());
195
196 final List<FunctionScoreQueryBuilder.FilterFunctionBuilder> flist = new ArrayList<>();
197 flist.add(new FunctionScoreQueryBuilder.FilterFunctionBuilder(ScoreFunctionBuilders.randomFunction(System
198 .currentTimeMillis())));
199 reader.setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.matchAllQuery(),
200 flist.toArray(new FunctionScoreQueryBuilder.FilterFunctionBuilder[flist.size()])).boostMode(
201 CombineFunction.MULTIPLY));
202 reader.addSort(SortBuilders.fieldSort(fessConfig.getIndexFieldClickCount()));
203 reader.addSort(SortBuilders.scoreSort());
204 return reader;
205 }, 2, fessConfig.getSuggestUpdateRequestIntervalAsInteger().longValue()).then(response -> {
206 refresh();
207 success.accept(true);
208 }).error(t -> error.accept(t));
209 }
210
211 public void purgeDocumentSuggest(final LocalDateTime time) {
212 final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
213 boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(
214 time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()));
215
216 boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()));
217 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
218 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
219
220 SuggestUtil.deleteByQuery(fessEsClient, suggester.getIndex(), suggester.getType(), boolQueryBuilder);
221 }
222
223 public void purgeSearchlogSuggest(final LocalDateTime time) {
224 final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
225 boolQueryBuilder.must(QueryBuilders.rangeQuery(FieldNames.TIMESTAMP).lt(
226 time.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli()));
227
228 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.DOCUMENT.toString()));
229 boolQueryBuilder.must(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.QUERY.toString()));
230 boolQueryBuilder.mustNot(QueryBuilders.termQuery(FieldNames.KINDS, SuggestItem.Kind.USER.toString()));
231
232 SuggestUtil.deleteByQuery(fessEsClient, suggester.getIndex(), suggester.getType(), boolQueryBuilder);
233 }
234
235 public long getAllWordsNum() {
236 return suggester.getAllWordsNum();
237 }
238
239 public long getDocumentWordsNum() {
240 return suggester.getDocumentWordsNum();
241 }
242
243 public long getQueryWordsNum() {
244 return suggester.getQueryWordsNum();
245 }
246
247 public boolean deleteAllWords() {
248 final SuggestDeleteResponse response = suggester.indexer().deleteAll();
249 if (response.hasError()) {
250 logger.warn("Failed to delete all words.", response.getErrors().get(0));
251 return false;
252 }
253 refresh();
254 return true;
255 }
256
257 public boolean deleteDocumentWords() {
258 final SuggestDeleteResponse response = suggester.indexer().deleteDocumentWords();
259 if (response.hasError()) {
260 logger.warn("Failed to delete document words.", response.getErrors().get(0));
261 return false;
262 }
263 refresh();
264 return true;
265 }
266
267 public boolean deleteQueryWords() {
268 final SuggestDeleteResponse response = suggester.indexer().deleteQueryWords();
269 if (response.hasError()) {
270 logger.warn("Failed to delete query words.", response.getErrors().get(0));
271 return false;
272 }
273 refresh();
274 return true;
275 }
276
277 public void storeAllElevateWords(final boolean apply) {
278 deleteAllElevateWord(apply);
279
280 final List<ElevateWord> list = elevateWordBhv.selectList(cb -> {
281 cb.query().matchAll();
282 cb.fetchFirst(ComponentUtil.getFessConfig().getPageElevateWordMaxFetchSizeAsInteger());
283 });
284
285 for (final ElevateWord elevateWord : list) {
286 addElevateWord(elevateWord.getSuggestWord(), elevateWord.getReading(), elevateWord.getLabelTypeValues(),
287 elevateWord.getPermissions(), elevateWord.getBoost(), apply);
288 }
289 refresh();
290 }
291
292 public void deleteAllElevateWord(final boolean apply) {
293 final List<ElevateWord> list = elevateWordBhv.selectList(cb -> {
294 cb.query().matchAll();
295 cb.fetchFirst(ComponentUtil.getFessConfig().getPageElevateWordMaxFetchSizeAsInteger());
296 });
297
298 for (final ElevateWord elevateWord : list) {
299 suggester.indexer().deleteElevateWord(elevateWord.getSuggestWord(), apply);
300 }
301 refresh();
302 }
303
304 public void deleteElevateWord(final String word, final boolean apply) {
305 suggester.indexer().deleteElevateWord(word, apply);
306 refresh();
307 }
308
309 public void addElevateWord(final String word, final String reading, final String[] tags, final String[] permissions, final Float boost,
310 final boolean apply) {
311 final String[] readings;
312 if (StringUtil.isBlank(reading)) {
313 readings = word.replace(" ", TEXT_SEP).replaceAll(TEXT_SEP + "+", TEXT_SEP).split(TEXT_SEP);
314 } else {
315 readings = reading.replace(" ", TEXT_SEP).replaceAll(TEXT_SEP + "+", TEXT_SEP).split(TEXT_SEP);
316 }
317
318 final List<String> labelList = new ArrayList<>();
319 if (tags != null) {
320 for (final String label : tags) {
321 labelList.add(label);
322 }
323 }
324 final List<String> roleList = new ArrayList<>();
325 if (permissions != null) {
326 for (final String permission : permissions) {
327 roleList.add(permission);
328 }
329 }
330
331 suggester.indexer().addElevateWord(
332 new org.codelibs.fess.suggest.entity.ElevateWord(word, boost, Arrays.asList(readings), contentFieldList, labelList,
333 roleList), apply);
334
335 refresh();
336 }
337
338 protected void deleteAllBadWords() {
339 suggester.settings().badword().deleteAll();
340 }
341
342 public void storeAllBadWords(final boolean apply) {
343 deleteAllBadWords();
344 final List<BadWord> list = badWordBhv.selectList(cb -> {
345 cb.query().matchAll();
346 cb.fetchFirst(ComponentUtil.getFessConfig().getPageBadWordMaxFetchSizeAsInteger());
347 });
348 for (final BadWord badWord : list) {
349 final String word = badWord.getSuggestWord();
350 suggester.indexer().addBadWord(word, apply);
351 }
352 refresh();
353 }
354
355 public void addBadWord(final String badWord, final boolean apply) {
356 suggester.indexer().addBadWord(badWord, apply);
357 refresh();
358 }
359
360 public void deleteBadWord(final String badWord) {
361 suggester.indexer().deleteBadWord(badWord);
362 refresh();
363 }
364
365 public synchronized void refresh() {
366 suggester.refresh();
367 if (popularWordHelper != null) {
368 popularWordHelper.clearCache();
369 }
370 }
371 }