1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.time.LocalDateTime;
21 import java.util.ArrayList;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Queue;
26 import java.util.concurrent.ConcurrentLinkedQueue;
27
28 import javax.annotation.PostConstruct;
29 import javax.servlet.http.HttpServletRequest;
30
31 import org.apache.commons.lang3.StringUtils;
32 import org.codelibs.core.collection.LruHashMap;
33 import org.codelibs.core.lang.StringUtil;
34 import org.codelibs.fess.Constants;
35 import org.codelibs.fess.app.service.SearchService;
36 import org.codelibs.fess.entity.SearchRequestParams;
37 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
38 import org.codelibs.fess.es.log.exbhv.ClickLogBhv;
39 import org.codelibs.fess.es.log.exbhv.FavoriteLogBhv;
40 import org.codelibs.fess.es.log.exbhv.SearchLogBhv;
41 import org.codelibs.fess.es.log.exbhv.UserInfoBhv;
42 import org.codelibs.fess.es.log.exentity.ClickLog;
43 import org.codelibs.fess.es.log.exentity.SearchLog;
44 import org.codelibs.fess.es.log.exentity.UserInfo;
45 import org.codelibs.fess.mylasta.action.FessUserBean;
46 import org.codelibs.fess.mylasta.direction.FessConfig;
47 import org.codelibs.fess.util.ComponentUtil;
48 import org.codelibs.fess.util.DocumentUtil;
49 import org.codelibs.fess.util.QueryResponseList;
50 import org.dbflute.optional.OptionalThing;
51 import org.elasticsearch.action.update.UpdateRequest;
52 import org.elasticsearch.script.Script;
53 import org.lastaflute.web.util.LaRequestUtil;
54 import org.slf4j.Logger;
55 import org.slf4j.LoggerFactory;
56
57 public class SearchLogHelper {
58 private static final Logger logger = LoggerFactory.getLogger(SearchLogHelper.class);
59
60 public long userCheckInterval = 5 * 60 * 1000L;
61
62 public int userInfoCacheSize = 1000;
63
64 protected volatile Queue<SearchLog> searchLogQueue = new ConcurrentLinkedQueue<>();
65
66 protected volatile Queue<ClickLog> clickLogQueue = new ConcurrentLinkedQueue<>();
67
68 protected Map<String, Long> userInfoCache;
69
70 @PostConstruct
71 public void init() {
72 userInfoCache = new LruHashMap<>(userInfoCacheSize);
73 }
74
75 public void addSearchLog(final SearchRequestParams params, final LocalDateTime requestedTime, final String queryId, final String query,
76 final int pageStart, final int pageSize, final QueryResponseList queryResponseList) {
77
78 final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
79 final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
80 final SearchLog searchLog = new SearchLog();
81
82 if (ComponentUtil.getFessConfig().isUserInfo()) {
83 final String userCode = userInfoHelper.getUserCode();
84 if (userCode != null) {
85 searchLog.setUserSessionId(userCode);
86 }
87 }
88
89 searchLog.setRoles(roleQueryHelper.build(params.getType()).stream().toArray(n -> new String[n]));
90 searchLog.setQueryId(queryId);
91 searchLog.setHitCount(queryResponseList.getAllRecordCount());
92 searchLog.setResponseTime(queryResponseList.getExecTime());
93 searchLog.setQueryTime(queryResponseList.getQueryTime());
94 searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
95 searchLog.setRequestedAt(requestedTime);
96 searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
97 searchLog.setQueryOffset(pageStart);
98 searchLog.setQueryPageSize(pageSize);
99 ComponentUtil.getRequestManager().findUserBean(FessUserBean.class).ifPresent(user -> {
100 searchLog.setUser(user.getUserId());
101 });
102
103 final HttpServletRequest request = LaRequestUtil.getRequest();
104 searchLog.setClientIp(StringUtils.abbreviate(ComponentUtil.getViewHelper().getClientIp(request), 100));
105 searchLog.setReferer(StringUtils.abbreviate(request.getHeader("referer"), 1000));
106 searchLog.setUserAgent(StringUtils.abbreviate(request.getHeader("user-agent"), 255));
107 final Object accessType = request.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
108 if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {
109 searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
110 } else if (Constants.SEARCH_LOG_ACCESS_TYPE_XML.equals(accessType)) {
111 searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_XML);
112 } else if (Constants.SEARCH_LOG_ACCESS_TYPE_OTHER.equals(accessType)) {
113 searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_OTHER);
114 } else {
115 searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_WEB);
116 }
117 final Object languages = request.getAttribute(Constants.REQUEST_LANGUAGES);
118 if (languages != null) {
119 searchLog.setLanguages(StringUtils.join((String[]) languages, ","));
120 } else {
121 searchLog.setLanguages(StringUtil.EMPTY);
122 }
123 final String virtualHostKey = ComponentUtil.getFessConfig().getVirtualHostKey();
124 if (virtualHostKey != null) {
125 searchLog.setVirtualHost(virtualHostKey);
126 } else {
127 searchLog.setVirtualHost(StringUtil.EMPTY);
128 }
129
130 @SuppressWarnings("unchecked")
131 final Map<String, List<String>> fieldLogMap = (Map<String, List<String>>) request.getAttribute(Constants.FIELD_LOGS);
132 if (fieldLogMap != null) {
133 final int queryMaxLength = ComponentUtil.getFessConfig().getQueryMaxLengthAsInteger();
134 for (final Map.Entry<String, List<String>> logEntry : fieldLogMap.entrySet()) {
135 for (final String value : logEntry.getValue()) {
136 searchLog.addSearchFieldLogValue(logEntry.getKey(), StringUtils.abbreviate(value, queryMaxLength));
137 }
138 }
139 }
140
141 searchLogQueue.add(searchLog);
142 }
143
144 public void addClickLog(final ClickLog clickLog) {
145 clickLogQueue.add(clickLog);
146 }
147
148 public void storeSearchLog() {
149 if (!searchLogQueue.isEmpty()) {
150 final Queue<SearchLog> queue = searchLogQueue;
151 searchLogQueue = new ConcurrentLinkedQueue<>();
152 processSearchLogQueue(queue);
153 }
154
155 if (!clickLogQueue.isEmpty()) {
156 final Queue<ClickLog> queue = clickLogQueue;
157 clickLogQueue = new ConcurrentLinkedQueue<>();
158 processClickLogQueue(queue);
159 }
160 }
161
162 public int getClickCount(final String url) {
163 final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
164 return clickLogBhv.selectCount(cb -> {
165 cb.query().setUrl_Equal(url);
166 });
167 }
168
169 public long getFavoriteCount(final String url) {
170 final FavoriteLogBhv favoriteLogBhv = ComponentUtil.getComponent(FavoriteLogBhv.class);
171 return favoriteLogBhv.selectCount(cb -> {
172 cb.query().setUrl_Equal(url);
173 });
174 }
175
176 public void updateUserInfo(final String userCode) {
177 final long current = System.currentTimeMillis();
178 final Long time = userInfoCache.get(userCode);
179 if (time == null || current - time.longValue() > userCheckInterval) {
180
181 final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
182
183 final LocalDateTime now = ComponentUtil.getSystemHelper().getCurrentTimeAsLocalDateTime();
184 userInfoBhv.selectByPK(userCode).ifPresent(userInfo -> {
185 userInfo.setUpdatedAt(now);
186 new Thread(() -> {
187 userInfoBhv.insertOrUpdate(userInfo);
188 }).start();
189 }).orElse(() -> {
190 final UserInfo userInfo = new UserInfo();
191 userInfo.setId(userCode);
192 userInfo.setCreatedAt(now);
193 userInfo.setUpdatedAt(now);
194 userInfoBhv.insert(userInfo);
195 });
196 userInfoCache.put(userCode, current);
197 }
198 }
199
200 protected void processSearchLogQueue(final Queue<SearchLog> queue) {
201 final FessConfig fessConfig = ComponentUtil.getFessConfig();
202 final String value = fessConfig.getPurgeByBots();
203 String[] botNames;
204 if (StringUtil.isBlank(value)) {
205 botNames = StringUtil.EMPTY_STRINGS;
206 } else {
207 botNames = value.split(",");
208 }
209
210 final List<SearchLog> searchLogList = new ArrayList<>();
211 final Map<String, UserInfo> userInfoMap = new HashMap<>();
212 queue.stream().forEach(
213 searchLog -> {
214 final String userAgent = searchLog.getUserAgent();
215 final boolean isBot =
216 userAgent != null
217 && stream(botNames).get(stream -> stream.anyMatch(botName -> userAgent.indexOf(botName) >= 0));
218 if (!isBot) {
219 searchLog.getUserInfo().ifPresent(userInfo -> {
220 final String code = userInfo.getId();
221 final UserInfo oldUserInfo = userInfoMap.get(code);
222 if (oldUserInfo != null) {
223 userInfo.setCreatedAt(oldUserInfo.getCreatedAt());
224 }
225 userInfoMap.put(code, userInfo);
226 });
227 searchLogList.add(searchLog);
228 }
229 });
230
231 if (!userInfoMap.isEmpty()) {
232 final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
233 final List<UserInfo> updateList = new ArrayList<>();
234 final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
235 userInfoBhv.selectList(cb -> {
236 cb.query().setId_InScope(userInfoMap.keySet());
237 cb.fetchFirst(userInfoMap.size());
238 }).forEach(userInfo -> {
239 final String code = userInfo.getId();
240 final UserInfo entity = userInfoMap.get(code);
241 entity.setId(userInfo.getId());
242 entity.setCreatedAt(userInfo.getCreatedAt());
243 updateList.add(entity);
244 insertList.remove(entity);
245 });
246 userInfoBhv.batchInsert(insertList);
247 userInfoBhv.batchUpdate(updateList);
248 searchLogList.stream().forEach(searchLog -> {
249 searchLog.getUserInfo().ifPresent(userInfo -> {
250 searchLog.setUserInfoId(userInfo.getId());
251 });
252 });
253 }
254
255 if (!searchLogList.isEmpty()) {
256 storeSearchLogList(searchLogList);
257 if (fessConfig.isSuggestSearchLog()) {
258 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
259 suggestHelper.indexFromSearchLog(searchLogList);
260 }
261 }
262 }
263
264 private void storeSearchLogList(final List<SearchLog> searchLogList) {
265 final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
266 searchLogBhv.batchUpdate(searchLogList, op -> {
267 op.setRefreshPolicy(Constants.TRUE);
268 });
269 }
270
271 protected void processClickLogQueue(final Queue<ClickLog> queue) {
272 final Map<String, Integer> clickCountMap = new HashMap<>();
273 final List<ClickLog> clickLogList = new ArrayList<>();
274 for (final ClickLog clickLog : queue) {
275 try {
276 final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
277 searchLogBhv.selectEntity(cb -> {
278 cb.query().setQueryId_Equal(clickLog.getQueryId());
279 }).ifPresent(entity -> {
280 clickLogList.add(clickLog);
281 final String docId = clickLog.getDocId();
282 Integer countObj = clickCountMap.get(docId);
283 if (countObj == null) {
284 countObj = Integer.valueOf(1);
285 } else {
286 countObj = countObj.intValue() + 1;
287 }
288 clickCountMap.put(docId, countObj);
289 }).orElse(() -> {
290 logger.warn("Not Found for SearchLog: " + clickLog);
291 });
292 } catch (final Exception e) {
293 logger.warn("Failed to process: " + clickLog, e);
294 }
295 }
296 if (!clickLogList.isEmpty()) {
297 try {
298 final ClickLogBhv clickLogBhv = ComponentUtil.getComponent(ClickLogBhv.class);
299 clickLogBhv.batchInsert(clickLogList);
300 } catch (final Exception e) {
301 logger.warn("Failed to insert: " + clickLogList, e);
302 }
303 }
304
305 if (!clickCountMap.isEmpty()) {
306 final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
307 try {
308 searchService.bulkUpdate(builder -> {
309 final FessConfig fessConfig = ComponentUtil.getFessConfig();
310 searchService.getDocumentListByDocIds(clickCountMap.keySet().toArray(new String[clickCountMap.size()]),
311 new String[] { fessConfig.getIndexFieldDocId() }, OptionalThing.of(FessUserBean.empty()),
312 SearchRequestType.ADMIN_SEARCH).forEach(
313 doc -> {
314 final String id = DocumentUtil.getValue(doc, fessConfig.getIndexFieldId(), String.class);
315 final String docId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
316 if (id != null && docId != null && clickCountMap.containsKey(docId)) {
317 final Integer count = clickCountMap.get(docId);
318 final Script script =
319 new Script("ctx._source." + fessConfig.getIndexFieldClickCount() + "+=" + count.toString());
320 final Map<String, Object> upsertMap = new HashMap<>();
321 upsertMap.put(fessConfig.getIndexFieldClickCount(), count);
322 builder.add(new UpdateRequest(fessConfig.getIndexDocumentUpdateIndex(), fessConfig
323 .getIndexDocumentType(), id).script(script).upsert(upsertMap));
324 }
325 });
326 });
327 } catch (final Exception e) {
328 logger.warn("Failed to update clickCounts", e);
329 }
330 }
331 }
332 }