1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import java.io.UnsupportedEncodingException;
19 import java.net.URLEncoder;
20 import java.util.ArrayList;
21 import java.util.Base64;
22 import java.util.Collections;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.LinkedHashMap;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.stream.Collectors;
29
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.codelibs.core.lang.StringUtil;
33 import org.codelibs.core.security.MessageDigestUtil;
34 import org.codelibs.fesen.index.query.QueryBuilders;
35 import org.codelibs.fesen.search.aggregations.AggregationBuilders;
36 import org.codelibs.fesen.search.aggregations.BucketOrder;
37 import org.codelibs.fesen.search.aggregations.bucket.terms.Terms;
38 import org.codelibs.fesen.search.aggregations.bucket.terms.Terms.Bucket;
39 import org.codelibs.fesen.search.aggregations.bucket.terms.TermsAggregationBuilder;
40 import org.codelibs.fess.Constants;
41 import org.codelibs.fess.app.service.CrawlingInfoService;
42 import org.codelibs.fess.es.client.SearchEngineClient;
43 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
44 import org.codelibs.fess.es.config.exentity.CrawlingInfo;
45 import org.codelibs.fess.es.config.exentity.CrawlingInfoParam;
46 import org.codelibs.fess.exception.FessSystemException;
47 import org.codelibs.fess.mylasta.direction.FessConfig;
48 import org.codelibs.fess.util.ComponentUtil;
49
50 public class CrawlingInfoHelper {
51 private static final Logger logger = LogManager.getLogger(CrawlingInfoHelper.class);
52
53 public static final String FACET_COUNT_KEY = "count";
54
55 protected Map<String, String> infoMap;
56
57 protected Long documentExpires;
58
59 protected int maxSessionIdsInList;
60
61 protected CrawlingInfoService getCrawlingInfoService() {
62 return ComponentUtil.getComponent(CrawlingInfoService.class);
63 }
64
65 public String getCanonicalSessionId(final String sessionId) {
66 final int idx = sessionId.indexOf('-');
67 if (idx >= 0) {
68 return sessionId.substring(0, idx);
69 }
70 return sessionId;
71 }
72
73 public synchronized void store(final String sessionId, final boolean create) {
74 CrawlingInfo crawlingInfo = create ? null : getCrawlingInfoService().getLast(sessionId);
75 if (crawlingInfo == null) {
76 crawlingInfo = new CrawlingInfo(sessionId);
77 try {
78 getCrawlingInfoService().store(crawlingInfo);
79 } catch (final Exception e) {
80 throw new FessSystemException("No crawling session.", e);
81 }
82 }
83
84 if (infoMap != null) {
85 final List<CrawlingInfoParam> crawlingInfoParamList = new ArrayList<>();
86 for (final Map.Entry<String, String> entry : infoMap.entrySet()) {
87 final CrawlingInfoParam crawlingInfoParam = new CrawlingInfoParam();
88 crawlingInfoParam.setCrawlingInfoId(crawlingInfo.getId());
89 crawlingInfoParam.setKey(entry.getKey());
90 crawlingInfoParam.setValue(entry.getValue());
91 crawlingInfoParamList.add(crawlingInfoParam);
92 }
93 getCrawlingInfoService().storeInfo(crawlingInfoParamList);
94 }
95
96 infoMap = null;
97 }
98
99 public synchronized void putToInfoMap(final String key, final String value) {
100 if (infoMap == null) {
101 infoMap = Collections.synchronizedMap(new LinkedHashMap<String, String>());
102 }
103 logger.debug("infoMap: {}={} => {}", key, value, infoMap);
104 infoMap.put(key, value);
105 }
106
107 public void updateParams(final String sessionId, final String name, final int dayForCleanup) {
108 final CrawlingInfo crawlingInfo = getCrawlingInfoService().getLast(sessionId);
109 if (crawlingInfo == null) {
110 logger.warn("No crawling session: {}", sessionId);
111 return;
112 }
113 if (StringUtil.isNotBlank(name)) {
114 crawlingInfo.setName(name);
115 } else {
116 crawlingInfo.setName(Constants.CRAWLING_INFO_SYSTEM_NAME);
117 }
118 if (dayForCleanup >= 0) {
119 final long expires = getExpiredTime(dayForCleanup);
120 crawlingInfo.setExpiredTime(expires);
121 documentExpires = expires;
122 }
123 try {
124 getCrawlingInfoService().store(crawlingInfo);
125 } catch (final Exception e) {
126 throw new FessSystemException("No crawling session.", e);
127 }
128
129 }
130
131 public Date getDocumentExpires(final CrawlingConfig config) {
132 if (config != null) {
133 final Integer timeToLive = config.getTimeToLive();
134 if (timeToLive != null) {
135
136 final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
137 return new Date(now + timeToLive.longValue() * 1000 * 60);
138 }
139 }
140 return documentExpires != null ? new Date(documentExpires) : null;
141 }
142
143 protected long getExpiredTime(final int days) {
144 final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
145 return now + days * Constants.ONE_DAY_IN_MILLIS;
146 }
147
148 public Map<String, String> getInfoMap(final String sessionId) {
149 final List<CrawlingInfoParam> crawlingInfoParamList = getCrawlingInfoService().getLastCrawlingInfoParamList(sessionId);
150 final Map<String, String> map = new HashMap<>();
151 for (final CrawlingInfoParam crawlingInfoParam : crawlingInfoParamList) {
152 map.put(crawlingInfoParam.getKey(), crawlingInfoParam.getValue());
153 }
154 return map;
155 }
156
157 public String generateId(final Map<String, Object> dataMap) {
158 final FessConfig fessConfig = ComponentUtil.getFessConfig();
159 final String url = (String) dataMap.get(fessConfig.getIndexFieldUrl());
160 final StringBuilder buf = new StringBuilder(1000);
161
162 @SuppressWarnings("unchecked")
163 final List<String> roleTypeList = (List<String>) dataMap.get(fessConfig.getIndexFieldRole());
164 buf.append(url);
165 if (roleTypeList != null && !roleTypeList.isEmpty()) {
166 buf.append(";r=");
167 buf.append(roleTypeList.stream().sorted().collect(Collectors.joining(",")));
168 }
169
170 @SuppressWarnings("unchecked")
171 final List<String> virtualHostList = (List<String>) dataMap.get(fessConfig.getIndexFieldVirtualHost());
172 if (virtualHostList != null && !virtualHostList.isEmpty()) {
173 buf.append(";v=");
174 buf.append(virtualHostList.stream().sorted().collect(Collectors.joining(",")));
175 }
176
177 final String urlId = buf.toString().trim();
178 return generateId(urlId);
179 }
180
181 public List<Map<String, String>> getSessionIdList(final SearchEngineClient searchEngineClient) {
182 final FessConfig fessConfig = ComponentUtil.getFessConfig();
183 return searchEngineClient.search(fessConfig.getIndexDocumentSearchIndex(), queryRequestBuilder -> {
184 queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
185 final TermsAggregationBuilder termsBuilder = AggregationBuilders.terms(fessConfig.getIndexFieldSegment())
186 .field(fessConfig.getIndexFieldSegment()).size(maxSessionIdsInList).order(BucketOrder.key(false));
187 queryRequestBuilder.addAggregation(termsBuilder);
188 queryRequestBuilder.setPreference(Constants.SEARCH_PREFERENCE_LOCAL);
189 return true;
190 }, (queryRequestBuilder, execTime, searchResponse) -> {
191 final List<Map<String, String>> sessionIdList = new ArrayList<>();
192 searchResponse.ifPresent(response -> {
193 final Terms terms = response.getAggregations().get(fessConfig.getIndexFieldSegment());
194 for (final Bucket bucket : terms.getBuckets()) {
195 final Map<String, String> map = new HashMap<>(2);
196 map.put(fessConfig.getIndexFieldSegment(), bucket.getKey().toString());
197 map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
198 sessionIdList.add(map);
199 }
200 });
201 return sessionIdList;
202 });
203 }
204
205 protected String generateId(final String urlId) {
206 final StringBuilder encodedBuf = new StringBuilder(urlId.length() + 100);
207 for (int i = 0; i < urlId.length(); i++) {
208 final char c = urlId.charAt(i);
209 if (c >= 'a' && c <= 'z'
210 || c >= 'A' && c <= 'Z'
211 || c >= '0' && c <= '9'
212 || c == '.'
213 || c == '-'
214 || c == '*'
215 || c == '_'
216 || c == ':'
217 || c == '+'
218 || c == '%'
219 || c == '='
220 || c == '&'
221 || c == '?'
222 || c == '#'
223 || c == '['
224 || c == ']'
225 || c == '@'
226 || c == '~'
227 || c == '!'
228 || c == '$'
229 || c == '\''
230 || c == '('
231 || c == ')'
232 || c == ','
233 || c == ';'
234 ) {
235 encodedBuf.append(c);
236 } else {
237 try {
238 final String target = String.valueOf(c);
239 final String converted = URLEncoder.encode(target, Constants.UTF_8);
240 if (target.equals(converted)) {
241 encodedBuf.append(Base64.getUrlEncoder().encodeToString(target.getBytes(Constants.CHARSET_UTF_8)));
242 } else {
243 encodedBuf.append(converted);
244 }
245 } catch (final UnsupportedEncodingException e) {
246
247 }
248 }
249 }
250
251 final String id = encodedBuf.toString();
252 return MessageDigestUtil.digest(ComponentUtil.getFessConfig().getIndexIdDigestAlgorithm(), id);
253 }
254
255 public void setMaxSessionIdsInList(final int maxSessionIdsInList) {
256 this.maxSessionIdsInList = maxSessionIdsInList;
257 }
258 }