1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import java.util.ArrayList;
19 import java.util.Collections;
20 import java.util.List;
21 import java.util.Map;
22 import java.util.regex.Pattern;
23
24 import javax.annotation.Resource;
25
26 import org.codelibs.core.lang.StringUtil;
27 import org.codelibs.fess.Constants;
28 import org.codelibs.fess.app.service.BoostDocumentRuleService;
29 import org.codelibs.fess.app.service.FailureUrlService;
30 import org.codelibs.fess.app.service.FileAuthenticationService;
31 import org.codelibs.fess.app.service.FileConfigService;
32 import org.codelibs.fess.app.service.WebConfigService;
33 import org.codelibs.fess.crawler.Crawler;
34 import org.codelibs.fess.crawler.CrawlerContext;
35 import org.codelibs.fess.crawler.CrawlerStatus;
36 import org.codelibs.fess.crawler.interval.FessIntervalController;
37 import org.codelibs.fess.crawler.service.impl.EsDataService;
38 import org.codelibs.fess.crawler.service.impl.EsUrlFilterService;
39 import org.codelibs.fess.crawler.service.impl.EsUrlQueueService;
40 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
41 import org.codelibs.fess.es.config.exentity.FileConfig;
42 import org.codelibs.fess.es.config.exentity.WebConfig;
43 import org.codelibs.fess.indexer.IndexUpdater;
44 import org.codelibs.fess.mylasta.direction.FessConfig;
45 import org.codelibs.fess.util.ComponentUtil;
46 import org.slf4j.Logger;
47 import org.slf4j.LoggerFactory;
48
49 public class WebFsIndexHelper {
50
51 private static final Logger logger = LoggerFactory.getLogger(WebFsIndexHelper.class);
52
53 @Resource
54 public WebConfigService webConfigService;
55
56 @Resource
57 protected FileConfigService fileConfigService;
58
59 @Resource
60 protected FileAuthenticationService fileAuthenticationService;
61
62 @Resource
63 public FailureUrlService failureUrlService;
64
65 @Resource
66 protected BoostDocumentRuleService boostDocumentRuleService;
67
68 @Resource
69 protected CrawlingConfigHelper crawlingConfigHelper;
70
71 public long maxAccessCount = Long.MAX_VALUE;
72
73 public long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;
74
75 public int indexUpdaterPriority = Thread.MAX_PRIORITY;
76
77 public int crawlerPriority = Thread.NORM_PRIORITY;
78
79 private final List<Crawler> crawlerList = Collections.synchronizedList(new ArrayList<Crawler>());
80
81 public void crawl(final String sessionId, final List<String> webConfigIdList, final List<String> fileConfigIdList) {
82 final boolean runAll = webConfigIdList == null && fileConfigIdList == null;
83 final List<WebConfig> webConfigList;
84 if (runAll || webConfigIdList != null) {
85 webConfigList = webConfigService.getWebConfigListByIds(webConfigIdList);
86 } else {
87 webConfigList = Collections.emptyList();
88 }
89 final List<FileConfig> fileConfigList;
90 if (runAll || fileConfigIdList != null) {
91 fileConfigList = fileConfigService.getFileConfigListByIds(fileConfigIdList);
92 } else {
93 fileConfigList = Collections.emptyList();
94 }
95
96 if (webConfigList.isEmpty() && fileConfigList.isEmpty()) {
97
98 if (logger.isInfoEnabled()) {
99 logger.info("No crawling target urls.");
100 }
101 return;
102 }
103
104 doCrawl(sessionId, webConfigList, fileConfigList);
105 }
106
107 protected void doCrawl(final String sessionId, final List<WebConfig> webConfigList, final List<FileConfig> fileConfigList) {
108 final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();
109
110 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
111 final FessConfig fessConfig = ComponentUtil.getFessConfig();
112
113 final long startTime = System.currentTimeMillis();
114
115 final List<String> sessionIdList = new ArrayList<>();
116 crawlerList.clear();
117 final List<String> crawlerStatusList = new ArrayList<>();
118
119 for (final WebConfig webConfig : webConfigList) {
120 final String sid = crawlingConfigHelper.store(sessionId, webConfig);
121
122
123 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
124 crawler.setSessionId(sid);
125 sessionIdList.add(sid);
126
127 final String urlsStr = webConfig.getUrls();
128 if (StringUtil.isBlank(urlsStr)) {
129 logger.warn("No target urls. Skipped");
130 break;
131 }
132
133
134 final int intervalTime =
135 webConfig.getIntervalTime() != null ? webConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;
136 ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
137
138 final String includedUrlsStr = webConfig.getIncludedUrls() != null ? webConfig.getIncludedUrls() : StringUtil.EMPTY;
139 final String excludedUrlsStr = webConfig.getExcludedUrls() != null ? webConfig.getExcludedUrls() : StringUtil.EMPTY;
140
141
142 final CrawlerContext crawlerContext = crawler.getCrawlerContext();
143 final int numOfThread =
144 webConfig.getNumOfThread() != null ? webConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
145 crawlerContext.setNumOfThread(numOfThread);
146
147
148 final int depth = webConfig.getDepth() != null ? webConfig.getDepth() : -1;
149 crawlerContext.setMaxDepth(depth);
150
151
152 final long maxCount = webConfig.getMaxAccessCount() != null ? webConfig.getMaxAccessCount() : maxAccessCount;
153 crawlerContext.setMaxAccessCount(maxCount);
154
155 webConfig.initializeClientFactory(crawler.getClientFactory());
156 final Map<String, String> configParamMap = webConfig.getConfigParameterMap(ConfigName.CONFIG);
157
158 if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_CLEANUP_ALL))) {
159 deleteCrawlData(sid);
160 } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_CLEANUP_FILTERS))) {
161 final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
162 try {
163 urlFilterService.delete(sid);
164 } catch (final Exception e) {
165 logger.warn("Failed to delete url filters for " + sid);
166 }
167 }
168
169
170 final String[] urls = urlsStr.split("[\r\n]");
171 for (final String u : urls) {
172 if (StringUtil.isNotBlank(u)) {
173 final String urlValue = u.trim();
174 if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(u)) {
175 crawler.addUrl(urlValue);
176 if (logger.isInfoEnabled()) {
177 logger.info("Target URL: " + urlValue);
178 }
179 }
180 }
181 }
182
183
184 final String[] includedUrls = includedUrlsStr.split("[\r\n]");
185 for (final String u : includedUrls) {
186 if (StringUtil.isNotBlank(u)) {
187 final String urlValue = u.trim();
188 if (!urlValue.startsWith("#")) {
189 crawler.addIncludeFilter(urlValue);
190 if (logger.isInfoEnabled()) {
191 logger.info("Included URL: " + urlValue);
192 }
193 }
194 }
195 }
196
197
198 final String[] excludedUrls = excludedUrlsStr.split("[\r\n]");
199 for (final String u : excludedUrls) {
200 if (StringUtil.isNotBlank(u)) {
201 final String urlValue = u.trim();
202 if (!urlValue.startsWith("#")) {
203 crawler.addExcludeFilter(urlValue);
204 if (logger.isInfoEnabled()) {
205 logger.info("Excluded URL: " + urlValue);
206 }
207 }
208 }
209 }
210
211
212 final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(webConfig.getConfigId());
213 for (final String u : excludedUrlList) {
214 if (StringUtil.isNotBlank(u)) {
215 final String urlValue = Pattern.quote(u.trim());
216 crawler.addExcludeFilter(urlValue);
217 if (logger.isInfoEnabled()) {
218 logger.info("Excluded URL from failures: " + urlValue);
219 }
220 }
221 }
222
223 if (logger.isDebugEnabled()) {
224 logger.debug("Crawling " + urlsStr);
225 }
226
227 crawler.setBackground(true);
228 crawler.setThreadPriority(crawlerPriority);
229
230 crawlerList.add(crawler);
231 crawlerStatusList.add(Constants.READY);
232 }
233
234
235 for (final FileConfig fileConfig : fileConfigList) {
236 final String sid = crawlingConfigHelper.store(sessionId, fileConfig);
237
238
239 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
240 crawler.setSessionId(sid);
241 sessionIdList.add(sid);
242
243 final String pathsStr = fileConfig.getPaths();
244 if (StringUtil.isBlank(pathsStr)) {
245 logger.warn("No target uris. Skipped");
246 break;
247 }
248
249 final int intervalTime =
250 fileConfig.getIntervalTime() != null ? fileConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_FS;
251 ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
252
253 final String includedPathsStr = fileConfig.getIncludedPaths() != null ? fileConfig.getIncludedPaths() : StringUtil.EMPTY;
254 final String excludedPathsStr = fileConfig.getExcludedPaths() != null ? fileConfig.getExcludedPaths() : StringUtil.EMPTY;
255
256
257 final CrawlerContext crawlerContext = crawler.getCrawlerContext();
258 final int numOfThread =
259 fileConfig.getNumOfThread() != null ? fileConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_FS;
260 crawlerContext.setNumOfThread(numOfThread);
261
262
263 final int depth = fileConfig.getDepth() != null ? fileConfig.getDepth() : -1;
264 crawlerContext.setMaxDepth(depth);
265
266
267 final long maxCount = fileConfig.getMaxAccessCount() != null ? fileConfig.getMaxAccessCount() : maxAccessCount;
268 crawlerContext.setMaxAccessCount(maxCount);
269
270 fileConfig.initializeClientFactory(crawler.getClientFactory());
271 final Map<String, String> configParamMap = fileConfig.getConfigParameterMap(ConfigName.CONFIG);
272
273 if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_CLEANUP_ALL))) {
274 deleteCrawlData(sid);
275 } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Constants.CONFIG_CLEANUP_FILTERS))) {
276 final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
277 try {
278 urlFilterService.delete(sid);
279 } catch (final Exception e) {
280 logger.warn("Failed to delete url filters for " + sid);
281 }
282 }
283
284
285 final String[] paths = pathsStr.split("[\r\n]");
286 for (String u : paths) {
287 if (StringUtil.isNotBlank(u)) {
288 u = u.trim();
289 if (!u.startsWith("#")) {
290 if (!fessConfig.isValidCrawlerFileProtocol(u)) {
291 if (u.startsWith("/")) {
292 u = "file:" + u;
293 } else {
294 u = "file:/" + u;
295 }
296 }
297 crawler.addUrl(u);
298 if (logger.isInfoEnabled()) {
299 logger.info("Target Path: " + u);
300 }
301 }
302 }
303 }
304
305
306 boolean urlEncodeDisabled = false;
307 final String[] includedPaths = includedPathsStr.split("[\r\n]");
308 for (final String u : includedPaths) {
309 if (StringUtil.isNotBlank(u)) {
310 final String line = u.trim();
311 if (!line.startsWith("#")) {
312 final String urlValue;
313 if (urlEncodeDisabled) {
314 urlValue = line;
315 urlEncodeDisabled = false;
316 } else {
317 urlValue = systemHelper.encodeUrlFilter(line);
318 }
319 crawler.addIncludeFilter(urlValue);
320 if (logger.isInfoEnabled()) {
321 logger.info("Included Path: " + urlValue);
322 }
323 } else if (line.startsWith("#DISABLE_URL_ENCODE")) {
324 urlEncodeDisabled = true;
325 }
326 }
327 }
328
329
330 urlEncodeDisabled = false;
331 final String[] excludedPaths = excludedPathsStr.split("[\r\n]");
332 for (final String u : excludedPaths) {
333 if (StringUtil.isNotBlank(u)) {
334 final String line = u.trim();
335 if (!line.startsWith("#")) {
336 final String urlValue;
337 if (urlEncodeDisabled) {
338 urlValue = line;
339 urlEncodeDisabled = false;
340 } else {
341 urlValue = systemHelper.encodeUrlFilter(line);
342 }
343 crawler.addExcludeFilter(urlValue);
344 if (logger.isInfoEnabled()) {
345 logger.info("Excluded Path: " + urlValue);
346 }
347 } else if (line.startsWith("#DISABLE_URL_ENCODE")) {
348 urlEncodeDisabled = true;
349 }
350 }
351 }
352
353
354 final List<String> excludedUrlList = failureUrlService.getExcludedUrlList(fileConfig.getConfigId());
355 if (excludedUrlList != null) {
356 for (final String u : excludedUrlList) {
357 if (StringUtil.isNotBlank(u)) {
358 final String urlValue = Pattern.quote(u.trim());
359 crawler.addExcludeFilter(urlValue);
360 if (logger.isInfoEnabled()) {
361 logger.info("Excluded Path from failures: " + urlValue);
362 }
363 }
364 }
365 }
366
367 if (logger.isDebugEnabled()) {
368 logger.debug("Crawling " + pathsStr);
369 }
370
371 crawler.setBackground(true);
372 crawler.setThreadPriority(crawlerPriority);
373
374 crawlerList.add(crawler);
375 crawlerStatusList.add(Constants.READY);
376 }
377
378
379 final IndexUpdater indexUpdater = ComponentUtil.getIndexUpdater();
380 indexUpdater.setName("IndexUpdater");
381 indexUpdater.setPriority(indexUpdaterPriority);
382 indexUpdater.setSessionIdList(sessionIdList);
383 indexUpdater.setDaemon(true);
384 indexUpdater.setCrawlerList(crawlerList);
385 boostDocumentRuleService.getAvailableBoostDocumentRuleList().forEach(rule -> {
386 indexUpdater.addDocBoostMatcher(new org.codelibs.fess.indexer.DocBoostMatcher(rule));
387 });
388 indexUpdater.start();
389
390 int startedCrawlerNum = 0;
391 int activeCrawlerNum = 0;
392 while (startedCrawlerNum < crawlerList.size()) {
393
394 if (systemHelper.isForceStop()) {
395 for (final Crawler crawler : crawlerList) {
396 crawler.stop();
397 }
398 break;
399 }
400
401 if (activeCrawlerNum < multiprocessCrawlingCount) {
402
403 crawlerList.get(startedCrawlerNum).execute();
404 crawlerStatusList.set(startedCrawlerNum, Constants.RUNNING);
405 startedCrawlerNum++;
406 activeCrawlerNum++;
407 try {
408 Thread.sleep(crawlingExecutionInterval);
409 } catch (final InterruptedException e) {
410 if (logger.isDebugEnabled()) {
411 logger.debug("Interrupted.", e);
412 }
413 }
414 continue;
415 }
416
417
418 for (int i = 0; i < startedCrawlerNum; i++) {
419 if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE
420 && crawlerStatusList.get(i).equals(Constants.RUNNING)) {
421 crawlerList.get(i).awaitTermination();
422 crawlerStatusList.set(i, Constants.DONE);
423 final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
424 indexUpdater.addFinishedSessionId(sid);
425 activeCrawlerNum--;
426 }
427 }
428 try {
429 Thread.sleep(crawlingExecutionInterval);
430 } catch (final InterruptedException e) {
431 if (logger.isDebugEnabled()) {
432 logger.debug("Interrupted.", e);
433 }
434 }
435 }
436
437 boolean finishedAll = false;
438 while (!finishedAll) {
439 finishedAll = true;
440 for (int i = 0; i < crawlerList.size(); i++) {
441 crawlerList.get(i).awaitTermination(crawlingExecutionInterval);
442 if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE
443 && !crawlerStatusList.get(i).equals(Constants.DONE)) {
444 crawlerStatusList.set(i, Constants.DONE);
445 final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
446 indexUpdater.addFinishedSessionId(sid);
447 }
448 if (!crawlerStatusList.get(i).equals(Constants.DONE)) {
449 finishedAll = false;
450 }
451 }
452 }
453 crawlerList.clear();
454 crawlerStatusList.clear();
455
456
457 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
458
459 final long execTime = System.currentTimeMillis() - startTime;
460 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_CRAWLING_EXEC_TIME, Long.toString(execTime));
461 if (logger.isInfoEnabled()) {
462 logger.info("[EXEC TIME] crawling time: " + execTime + "ms");
463 }
464
465 indexUpdater.setFinishCrawling(true);
466 try {
467 indexUpdater.join();
468 } catch (final InterruptedException e) {
469 logger.warn("Interrupted index update.", e);
470 }
471
472 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_EXEC_TIME, Long.toString(indexUpdater.getExecuteTime()));
473 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_SIZE, Long.toString(indexUpdater.getDocumentSize()));
474
475 if (systemHelper.isForceStop()) {
476 return;
477 }
478
479 for (final String sid : sessionIdList) {
480
481 crawlingConfigHelper.remove(sid);
482 deleteCrawlData(sid);
483 }
484 }
485
486 protected void deleteCrawlData(final String sid) {
487 final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
488 final EsUrlQueueService urlQueueService = ComponentUtil.getComponent(EsUrlQueueService.class);
489 final EsDataService dataService = ComponentUtil.getComponent(EsDataService.class);
490
491 try {
492
493 urlFilterService.delete(sid);
494 } catch (final Exception e) {
495 logger.warn("Failed to delete UrlFilter for " + sid, e);
496 }
497
498 try {
499
500 urlQueueService.clearCache();
501 urlQueueService.delete(sid);
502 } catch (final Exception e) {
503 logger.warn("Failed to delete UrlQueue for " + sid, e);
504 }
505
506 try {
507
508 dataService.delete(sid);
509 } catch (final Exception e) {
510 logger.warn("Failed to delete AccessResult for " + sid, e);
511 }
512 }
513
514 }