1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.util.ArrayList;
21 import java.util.Date;
22 import java.util.HashMap;
23 import java.util.HashSet;
24 import java.util.LinkedHashSet;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.Set;
28 import java.util.stream.Collectors;
29
30 import org.apache.commons.io.IOUtils;
31 import org.codelibs.core.lang.StringUtil;
32 import org.codelibs.fess.app.service.FailureUrlService;
33 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
34 import org.codelibs.fess.crawler.client.CrawlerClient;
35 import org.codelibs.fess.crawler.client.smb.SmbClient;
36 import org.codelibs.fess.crawler.entity.RequestData;
37 import org.codelibs.fess.crawler.entity.ResponseData;
38 import org.codelibs.fess.crawler.entity.UrlQueue;
39 import org.codelibs.fess.crawler.log.LogType;
40 import org.codelibs.fess.es.client.FessEsClient;
41 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
42 import org.codelibs.fess.exception.ContainerNotAvailableException;
43 import org.codelibs.fess.exception.ContentNotFoundException;
44 import org.codelibs.fess.helper.CrawlingConfigHelper;
45 import org.codelibs.fess.helper.CrawlingInfoHelper;
46 import org.codelibs.fess.helper.DuplicateHostHelper;
47 import org.codelibs.fess.helper.IndexingHelper;
48 import org.codelibs.fess.helper.SambaHelper;
49 import org.codelibs.fess.mylasta.direction.FessConfig;
50 import org.codelibs.fess.util.ComponentUtil;
51 import org.codelibs.fess.util.DocumentUtil;
52 import org.slf4j.Logger;
53 import org.slf4j.LoggerFactory;
54
55 import jcifs.smb.ACE;
56 import jcifs.smb.SID;
57
58 public class FessCrawlerThread extends CrawlerThread {
59 private static final Logger logger = LoggerFactory.getLogger(FessCrawlerThread.class);
60
61 @Override
62 protected boolean isContentUpdated(final CrawlerClient client, final UrlQueue<?> urlQueue) {
63 if (ComponentUtil.getFessConfig().isIncrementalCrawling()) {
64
65 final long startTime = System.currentTimeMillis();
66
67 final FessConfig fessConfig = ComponentUtil.getFessConfig();
68 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
69 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
70 final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
71 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
72 final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
73
74 final String url = urlQueue.getUrl();
75 ResponseData responseData = null;
76 try {
77 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(crawlerContext.getSessionId());
78 final Map<String, Object> dataMap = new HashMap<>();
79 dataMap.put(fessConfig.getIndexFieldUrl(), url);
80 final List<String> roleTypeList = new ArrayList<>();
81 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
82 if (url.startsWith("smb://")) {
83 if (url.endsWith("/")) {
84
85 return true;
86 }
87 if (fessConfig.isSmbRoleFromFile()) {
88
89 responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
90 if (responseData == null) {
91 return true;
92 }
93
94 final ACE[] aces = (ACE[]) responseData.getMetaDataMap().get(SmbClient.SMB_ACCESS_CONTROL_ENTRIES);
95 if (aces != null) {
96 for (final ACE item : aces) {
97 final SID sid = item.getSID();
98 final String accountId = sambaHelper.getAccountId(sid);
99 if (accountId != null) {
100 roleTypeList.add(accountId);
101 }
102 }
103 if (logger.isDebugEnabled()) {
104 logger.debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
105 }
106 }
107 }
108 }
109 dataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
110 final String id = crawlingInfoHelper.generateId(dataMap);
111
112 if (logger.isDebugEnabled()) {
113 logger.debug("Searching indexed document: " + id);
114 }
115 final Map<String, Object> document =
116 indexingHelper.getDocument(
117 fessEsClient,
118 id,
119 new String[] { fessConfig.getIndexFieldId(), fessConfig.getIndexFieldLastModified(),
120 fessConfig.getIndexFieldAnchor(), fessConfig.getIndexFieldSegment(),
121 fessConfig.getIndexFieldExpires(), fessConfig.getIndexFieldClickCount(),
122 fessConfig.getIndexFieldFavoriteCount() });
123 if (document == null) {
124 storeChildUrlsToQueue(urlQueue, getChildUrlSet(fessEsClient, id));
125 return true;
126 }
127
128 final Date expires = DocumentUtil.getValue(document, fessConfig.getIndexFieldExpires(), Date.class);
129 if (expires != null && expires.getTime() < System.currentTimeMillis()) {
130 final Object idValue = document.get(fessConfig.getIndexFieldId());
131 if (idValue != null && !indexingHelper.deleteDocument(fessEsClient, idValue.toString())) {
132 logger.debug("Failed to delete expired document: " + url);
133 }
134 return true;
135 }
136
137 final Date lastModified = DocumentUtil.getValue(document, fessConfig.getIndexFieldLastModified(), Date.class);
138 if (lastModified == null) {
139 return true;
140 }
141 urlQueue.setLastModified(lastModified.getTime());
142 log(logHelper, LogType.CHECK_LAST_MODIFIED, crawlerContext, urlQueue);
143
144 if (responseData == null) {
145
146 responseData = client.execute(RequestDataBuilder.newRequestData().head().url(url).build());
147 if (responseData == null) {
148 return true;
149 }
150 }
151
152 final int httpStatusCode = responseData.getHttpStatusCode();
153 if (logger.isDebugEnabled()) {
154 logger.debug("Accessing document: " + url + ", status: " + httpStatusCode);
155 }
156 if (httpStatusCode == 404) {
157 storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
158 if (!indexingHelper.deleteDocument(fessEsClient, id)) {
159 logger.debug("Failed to delete 404 document: " + url);
160 }
161 return false;
162 } else if (responseData.getLastModified() == null) {
163 return true;
164 } else if (responseData.getLastModified().getTime() <= lastModified.getTime() && httpStatusCode == 200) {
165
166 log(logHelper, LogType.NOT_MODIFIED, crawlerContext, urlQueue);
167
168 responseData.setExecutionTime(System.currentTimeMillis() - startTime);
169 responseData.setParentUrl(urlQueue.getParentUrl());
170 responseData.setSessionId(crawlerContext.getSessionId());
171 responseData.setHttpStatusCode(org.codelibs.fess.crawler.Constants.NOT_MODIFIED_STATUS);
172 processResponse(urlQueue, responseData);
173
174 storeChildUrlsToQueue(urlQueue, getAnchorSet(document.get(fessConfig.getIndexFieldAnchor())));
175
176 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
177 if (documentExpires != null
178 && !indexingHelper.updateDocument(fessEsClient, id, fessConfig.getIndexFieldExpires(), documentExpires)) {
179 logger.debug("Failed to update " + fessConfig.getIndexFieldExpires() + " at " + url);
180 }
181
182 return false;
183 }
184 } finally {
185 if (responseData != null) {
186 IOUtils.closeQuietly(responseData);
187 }
188 }
189 }
190 return true;
191 }
192
193 protected void storeChildUrlsToQueue(final UrlQueue<?> urlQueue, final Set<RequestData> childUrlSet) {
194 if (childUrlSet != null) {
195
196 try {
197 storeChildUrls(childUrlSet.stream().filter(rd -> StringUtil.isNotBlank(rd.getUrl())).collect(Collectors.toSet()),
198 urlQueue.getUrl(), urlQueue.getDepth() != null ? urlQueue.getDepth() + 1 : 1);
199 } catch (final Throwable t) {
200 if (!ComponentUtil.available()) {
201 throw new ContainerNotAvailableException(t);
202 }
203 throw t;
204 }
205 }
206 }
207
208 @SuppressWarnings("unchecked")
209 protected Set<RequestData> getAnchorSet(final Object obj) {
210 List<String> anchorList;
211 if (obj instanceof String) {
212 anchorList = new ArrayList<>();
213 anchorList.add(obj.toString());
214 } else if (obj instanceof List<?>) {
215 anchorList = (List<String>) obj;
216 } else {
217 return null;
218 }
219
220 if (anchorList.isEmpty()) {
221 return null;
222 }
223
224 final Set<RequestData> childUrlSet = new LinkedHashSet<>();
225 for (final String anchor : anchorList) {
226 childUrlSet.add(RequestDataBuilder.newRequestData().get().url(anchor).build());
227 }
228 return childUrlSet;
229 }
230
231 protected Set<RequestData> getChildUrlSet(final FessEsClient fessEsClient, final String id) {
232 final FessConfig fessConfig = ComponentUtil.getFessConfig();
233 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
234 final List<Map<String, Object>> docList =
235 indexingHelper.getChildDocumentList(fessEsClient, id, new String[] { fessConfig.getIndexFieldUrl() });
236 if (docList.isEmpty()) {
237 return null;
238 }
239 if (logger.isDebugEnabled()) {
240 logger.debug("Found documents: " + docList);
241 }
242 final Set<RequestData> urlSet = new HashSet<>(docList.size());
243 for (final Map<String, Object> doc : docList) {
244 final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
245 if (StringUtil.isNotBlank(url)) {
246 urlSet.add(RequestDataBuilder.newRequestData().get().url(url).build());
247 }
248 }
249 return urlSet;
250 }
251
252 @Override
253 protected void processResponse(final UrlQueue<?> urlQueue, final ResponseData responseData) {
254 super.processResponse(urlQueue, responseData);
255
256 final FessConfig fessConfig = ComponentUtil.getFessConfig();
257 if (fessConfig.isCrawlerFailureUrlStatusCodes(responseData.getHttpStatusCode())) {
258 final String sessionId = crawlerContext.getSessionId();
259 final CrawlingConfig crawlingConfig = ComponentUtil.getCrawlingConfigHelper().get(sessionId);
260 final String url = urlQueue.getUrl();
261
262 final FailureUrlService failureUrlService = ComponentUtil.getComponent(FailureUrlService.class);
263 failureUrlService.store(crawlingConfig, ContentNotFoundException.class.getCanonicalName(), url, new ContentNotFoundException(
264 url));
265 }
266 }
267
268 @Override
269 protected void storeChildUrl(final String childUrl, final String parentUrl, final String metaData, final int depth) {
270 if (StringUtil.isNotBlank(childUrl)) {
271 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
272 final String url = duplicateHostHelper.convert(childUrl);
273 super.storeChildUrl(url, parentUrl, metaData, depth);
274 }
275 }
276 }