1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.util.ArrayList;
21 import java.util.Date;
22 import java.util.HashMap;
23 import java.util.HashSet;
24 import java.util.LinkedHashSet;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Set;
28 import java.util.stream.Collectors;
29
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.codelibs.core.io.CloseableUtil;
33 import org.codelibs.core.lang.StringUtil;
34 import org.codelibs.fess.app.service.FailureUrlService;
35 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
36 import org.codelibs.fess.crawler.client.CrawlerClient;
37 import org.codelibs.fess.crawler.entity.RequestData;
38 import org.codelibs.fess.crawler.entity.ResponseData;
39 import org.codelibs.fess.crawler.entity.UrlQueue;
40 import org.codelibs.fess.crawler.log.LogType;
41 import org.codelibs.fess.es.client.SearchEngineClient;
42 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
43 import org.codelibs.fess.exception.ContainerNotAvailableException;
44 import org.codelibs.fess.exception.ContentNotFoundException;
45 import org.codelibs.fess.helper.CrawlingConfigHelper;
46 import org.codelibs.fess.helper.CrawlingInfoHelper;
47 import org.codelibs.fess.helper.DuplicateHostHelper;
48 import org.codelibs.fess.helper.IndexingHelper;
49 import org.codelibs.fess.helper.PermissionHelper;
50 import org.codelibs.fess.mylasta.direction.FessConfig;
51 import org.codelibs.fess.util.ComponentUtil;
52 import org.codelibs.fess.util.DocumentUtil;
53
54 public class FessCrawlerThread extends CrawlerThread {
55 private static final Logger logger = LogManager.getLogger(FessCrawlerThread.class);
56
57 @Override
58 protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {
59 if (ComponentUtil.getFessConfig().isIncrementalCrawling()) {
60
61 final long startTime = System.currentTimeMillis();
62
63 final FessConfig fessConfig = ComponentUtil.getFessConfig();
64 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
65 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
66 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
67 final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
68
69 final String url = urlQueue.getUrl();
70 ResponseData responseData = null;
71 try {
72 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(crawlerContext.getSessionId());
73 final Map<String, Object> dataMap = new HashMap<>();
74 dataMap.put(fessConfig.getIndexFieldUrl(), url);
75 final List<String> roleTypeList = new ArrayList<>();
76 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
77 if (url.startsWith("smb:") || url.startsWith("smb1:") || url.startsWith("file:") || url.startsWith("ftp:")) {
78 if (url.endsWith("/")) {
79
80 return true;
81 }
82 final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
83 if (fessConfig.isSmbRoleFromFile() || fessConfig.isFileRoleFromFile() || fessConfig.isFtpRoleFromFile()) {
84
85 responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
86 if (responseData == null) {
87 return true;
88 }
89
90 roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
91 roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
92 roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
93 }
94 }
95 dataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
96 final String id = crawlingInfoHelper.generateId(dataMap);
97
98 if (logger.isDebugEnabled()) {
99 logger.debug("Searching indexed document: {}", id);
100 }
101 final Map<String, Object> document = indexingHelper.getDocument(searchEngineClient, id,
102 new String[] { fessConfig.getIndexFieldId(), fessConfig.getIndexFieldLastModified(),
103 fessConfig.getIndexFieldAnchor(), fessConfig.getIndexFieldSegment(), fessConfig.getIndexFieldExpires(),
104 fessConfig.getIndexFieldClickCount(), fessConfig.getIndexFieldFavoriteCount() });
105 if (document == null) {
106 storeChildUrlsToQueue(urlQueue, getChildUrlSet(searchEngineClient, id));
107 return true;
108 }
109
110 final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);
111 if (expires != null && expires.getTime() < System.currentTimeMillis()) {
112 final Object idValue = document.get(fessConfig.getIndexFieldId());
113 if (idValue != null && !indexingHelper.deleteDocument(searchEngineClient, idValue.toString())) {
114 logger.debug("Failed to delete expired document: {}", url);
115 }
116 return true;
117 }
118
119 final Date lastModified = DocumentUtil.getValue(document, fessConfig.getIndexFieldLastModified(), Date.class);
120 if (lastModified == null) {
121 return true;
122 }
123 urlQueue.setLastModified(lastModified.getTime());
124 log(logHelper, LogType.CHECK_LAST_MODIFIED, crawlerContext, urlQueue);
125
126 if (responseData == null) {
127
128 responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
129 if (responseData == null) {
130 return true;
131 }
132 }
133
134 final int httpStatusCode = responseData.getHttpStatusCode();
135 if (logger.isDebugEnabled()) {
136 logger.debug("Accessing document: {}, status: {}", url, httpStatusCode);
137 }
138 if (httpStatusCode == 404) {
139 storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
140 if (!indexingHelper.deleteDocument(searchEngineClient, id)) {
141 logger.debug("Failed to delete 404 document: {}", url);
142 }
143 return false;
144 }
145 if (responseData.getLastModified() == null) {
146 return true;
147 }
148 if (responseData.getLastModified().getTime() <= lastModified.getTime() && httpStatusCode == 200) {
149
150 log(logHelper, LogType.NOT_MODIFIED, crawlerContext, urlQueue);
151
152 responseData.setExecutionTime(System.currentTimeMillis() - startTime);
153 responseData.setParentUrl(urlQueue.getParentUrl());
154 responseData.setSessionId(crawlerContext.getSessionId());
155 responseData.setHttpStatusCode(org.codelibs.fess.crawler.Constants.NOT_MODIFIED_STATUS);
156 processResponse(urlQueue, responseData);
157
158 storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
159
160 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
161 if (documentExpires != null
162 && !indexingHelper.updateDocument(searchEngineClient, id, fessConfig.getIndexFieldExpires(), documentExpires)) {
163 logger.debug("Failed to update {} at {}", fessConfig.getIndexFieldExpires(), url);
164 }
165
166 return false;
167 }
168 } finally {
169 if (responseData != null) {
170 CloseableUtil.closeQuietly(responseData);
171 }
172 }
173 }
174 return true;
175 }
176
177 protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) {
178 if (childUrlSet != null) {
179
180 try {
181 storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
182 urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
183 } catch (final Throwable t) {
184 if (!ComponentUtil.available()) {
185 throw new ContainerNotAvailableException(t);
186 }
187 throw t;
188 }
189 }
190 }
191
192 @SuppressWarnings("unchecked")
193 protected Set<RequestData> getAnchorSet(final Object obj) {
194 List<String> anchorList;
195 if (obj instanceof String) {
196 anchorList = new ArrayList<>();
197 anchorList.add(obj.toString());
198 } else if (obj instanceof List<?>) {
199 anchorList = (List<String>) obj;
200 } else {
201 return null;
202 }
203
204 if (anchorList.isEmpty()) {
205 return null;
206 }
207
208 final Set<RequestData> childUrlSet = new LinkedHashSet<>();
209 for (final String anchor : anchorList) {
210 childUrlSet.add(RequestDataBuilder.newRequestData().get().url(anchor).build());
211 }
212 return childUrlSet;
213 }
214
215 protected Set<RequestData> getChildUrlSet(final SearchEngineClient searchEngineClient, final String id) {
216 final FessConfig fessConfig = ComponentUtil.getFessConfig();
217 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
218 final List<Map<String, Object>> docList =
219 indexingHelper.getChildDocumentList(searchEngineClient, id, new String[] { fessConfig.getIndexFieldUrl() });
220 if (docList.isEmpty()) {
221 return null;
222 }
223 if (logger.isDebugEnabled()) {
224 logger.debug("Found documents: {}", docList);
225 }
226 final Set<RequestData> urlSet = new HashSet<>(docList.size());
227 for (final Map<String, Object> doc : docList) {
228 final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
229 if (StringUtil.isNotBlank(url)) {
230 urlSet.add(RequestDataBuilder.newRequestData().get().url(url).build());
231 }
232 }
233 return urlSet;
234 }
235
236 @Override
237 protected void processResponse(final UrlQueue<?> urlQueue, final ResponseData responseData) {
238 super.processResponse(urlQueue, responseData);
239
240 final FessConfig fessConfig = ComponentUtil.getFessConfig();
241 if (fessConfig.isCrawlerFailureUrlStatusCodes(responseData.getHttpStatusCode())) {
242 final String sessionId = crawlerContext.getSessionId();
243 final CrawlingConfig crawlingConfig = ComponentUtil.getCrawlingConfigHelper().get(sessionId);
244 final String url = urlQueue.getUrl();
245
246 final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
247 failureUrlService.store(crawlingConfig, ContentNotFoundException.class.getCanonicalName(), url,
248 new ContentNotFoundException(urlQueue.getParentUrl(), url));
249 }
250 }
251
252 @Override
253 protected void storeChildUrl(final String childUrl, final String parentUrl, final String metaData, final int depth) {
254 if (StringUtil.isNotBlank(childUrl)) {
255 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
256 final String url = duplicateHostHelper.convert(childUrl);
257 super.storeChildUrl(url, parentUrl, metaData, depth);
258 }
259 }
260 }