1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.util.ArrayList;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.concurrent.atomic.AtomicBoolean;
25 import java.util.regex.Pattern;
26
27 import org.apache.logging.log4j.LogManager;
28 import org.apache.logging.log4j.Logger;
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.core.lang.ThreadUtil;
31 import org.codelibs.fess.Constants;
32 import org.codelibs.fess.crawler.Crawler;
33 import org.codelibs.fess.crawler.CrawlerContext;
34 import org.codelibs.fess.crawler.CrawlerStatus;
35 import org.codelibs.fess.crawler.interval.FessIntervalController;
36 import org.codelibs.fess.crawler.service.impl.OpenSearchDataService;
37 import org.codelibs.fess.crawler.service.impl.OpenSearchUrlFilterService;
38 import org.codelibs.fess.crawler.service.impl.OpenSearchUrlQueueService;
39 import org.codelibs.fess.indexer.IndexUpdater;
40 import org.codelibs.fess.opensearch.config.exbhv.BoostDocumentRuleBhv;
41 import org.codelibs.fess.opensearch.config.exentity.BoostDocumentRule;
42 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.ConfigName;
43 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.Param.Config;
44 import org.codelibs.fess.opensearch.config.exentity.FileConfig;
45 import org.codelibs.fess.opensearch.config.exentity.WebConfig;
46 import org.codelibs.fess.util.ComponentUtil;
47
48
49
50
51
52
53 public class WebFsIndexHelper {
54
55
56
57
58 public WebFsIndexHelper() {
59
60 }
61
62 private static final Logger logger = LogManager.getLogger(WebFsIndexHelper.class);
63
64 private static final String DISABLE_URL_ENCODE = "#DISABLE_URL_ENCODE";
65
66
67
68
69 protected long maxAccessCount = Long.MAX_VALUE;
70
71
72
73
74 protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;
75
76
77
78
79 protected int indexUpdaterPriority = Thread.MAX_PRIORITY;
80
81
82
83
84 protected int crawlerPriority = Thread.NORM_PRIORITY;
85
86
87
88
89 protected final List<Crawler> crawlerList = Collections.synchronizedList(new ArrayList<>());
90
91
92
93
94
95
96
97
98 public void crawl(final String sessionId, final List<String> webConfigIdList, final List<String> fileConfigIdList) {
99 final boolean runAll = webConfigIdList == null && fileConfigIdList == null;
100 final List<WebConfig> webConfigList;
101 if (runAll || webConfigIdList != null) {
102 webConfigList = ComponentUtil.getCrawlingConfigHelper().getWebConfigListByIds(webConfigIdList);
103 } else {
104 webConfigList = Collections.emptyList();
105 }
106 final List<FileConfig> fileConfigList;
107 if (runAll || fileConfigIdList != null) {
108 fileConfigList = ComponentUtil.getCrawlingConfigHelper().getFileConfigListByIds(fileConfigIdList);
109 } else {
110 fileConfigList = Collections.emptyList();
111 }
112
113 if (webConfigList.isEmpty() && fileConfigList.isEmpty()) {
114
115 if (logger.isInfoEnabled()) {
116 logger.info("No crawling target urls.");
117 }
118 return;
119 }
120
121 doCrawl(sessionId, webConfigList, fileConfigList);
122 }
123
124
125
126
127
128
129
130
131 protected void doCrawl(final String sessionId, final List<WebConfig> webConfigList, final List<FileConfig> fileConfigList) {
132 final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();
133
134 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
135 ComponentUtil.getFessConfig();
136 final ProtocolHelper protocolHelper = ComponentUtil.getProtocolHelper();
137
138 final long startTime = systemHelper.getCurrentTimeAsLong();
139
140 final List<String> sessionIdList = new ArrayList<>();
141 crawlerList.clear();
142 final List<String> crawlerStatusList = new ArrayList<>();
143
144 for (final WebConfig webConfig : webConfigList) {
145 final String sid = ComponentUtil.getCrawlingConfigHelper().store(sessionId, webConfig);
146
147
148 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
149 crawler.setSessionId(sid);
150 sessionIdList.add(sid);
151
152 final String urlsStr = webConfig.getUrls();
153 if (StringUtil.isBlank(urlsStr)) {
154 logger.warn("[{}] No target urls. Skipped.", webConfig.getName());
155 break;
156 }
157
158
159 final int intervalTime =
160 webConfig.getIntervalTime() != null ? webConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;
161 ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
162
163 final String includedUrlsStr = webConfig.getIncludedUrls() != null ? webConfig.getIncludedUrls() : StringUtil.EMPTY;
164 final String excludedUrlsStr = webConfig.getExcludedUrls() != null ? webConfig.getExcludedUrls() : StringUtil.EMPTY;
165
166
167 final CrawlerContext crawlerContext = crawler.getCrawlerContext();
168 final int numOfThread =
169 webConfig.getNumOfThread() != null ? webConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
170 crawlerContext.setNumOfThread(numOfThread);
171
172
173 final int depth = webConfig.getDepth() != null ? webConfig.getDepth() : -1;
174 crawlerContext.setMaxDepth(depth);
175
176
177 final long maxCount = webConfig.getMaxAccessCount() != null ? webConfig.getMaxAccessCount() : maxAccessCount;
178 crawlerContext.setMaxAccessCount(maxCount);
179
180 webConfig.initializeClientFactory(() -> crawler.getClientFactory());
181 final Map<String, String> configParamMap = webConfig.getConfigParameterMap(ConfigName.CONFIG);
182
183 if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
184 deleteCrawlData(sid);
185 } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
186 final OpenSearchUrlFilterService urlFilterService = ComponentUtil.getComponent(OpenSearchUrlFilterService.class);
187 try {
188 urlFilterService.delete(sid);
189 } catch (final Exception e) {
190 logger.warn("Failed to delete UrlFilter: sessionId={}", sid);
191 }
192 }
193
194 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
195
196
197 split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
198 if (!urlValue.startsWith("#") && protocolHelper.isValidWebProtocol(urlValue)) {
199 final String u = duplicateHostHelper.convert(urlValue);
200 crawler.addUrl(u);
201 if (logger.isInfoEnabled()) {
202 logger.info("Target URL: {}", u);
203 }
204 }
205 }));
206
207
208 final AtomicBoolean urlEncodeDisabled = new AtomicBoolean(false);
209 split(includedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
210 if (!line.startsWith("#")) {
211 final String urlValue;
212 if (urlEncodeDisabled.get()) {
213 urlValue = line;
214 urlEncodeDisabled.set(false);
215 } else {
216 urlValue = systemHelper.encodeUrlFilter(line);
217 }
218 crawler.addIncludeFilter(urlValue);
219 if (logger.isInfoEnabled()) {
220 logger.info("Included URL: {}", urlValue);
221 }
222 } else if (line.startsWith(DISABLE_URL_ENCODE)) {
223 urlEncodeDisabled.set(true);
224 }
225 }));
226
227
228 urlEncodeDisabled.set(false);
229 split(excludedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
230 if (!line.startsWith("#")) {
231 final String urlValue;
232 if (urlEncodeDisabled.get()) {
233 urlValue = line;
234 urlEncodeDisabled.set(false);
235 } else {
236 urlValue = systemHelper.encodeUrlFilter(line);
237 }
238 crawler.addExcludeFilter(urlValue);
239 if (logger.isInfoEnabled()) {
240 logger.info("Excluded URL: {}", urlValue);
241 }
242 } else if (line.startsWith(DISABLE_URL_ENCODE)) {
243 urlEncodeDisabled.set(true);
244 }
245 }));
246
247
248 final List<String> excludedUrlList = ComponentUtil.getCrawlingConfigHelper().getExcludedUrlList(webConfig.getConfigId());
249 if (excludedUrlList != null) {
250 excludedUrlList.stream().filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(u -> {
251 final String urlValue = Pattern.quote(u);
252 crawler.addExcludeFilter(urlValue);
253 if (logger.isInfoEnabled()) {
254 logger.info("Excluded URL from failures: {}", urlValue);
255 }
256 });
257 }
258
259 if (logger.isDebugEnabled()) {
260 logger.debug("Crawling {}", urlsStr);
261 }
262
263 crawler.setBackground(true);
264 crawler.setThreadPriority(crawlerPriority);
265
266 crawlerList.add(crawler);
267 crawlerStatusList.add(Constants.READY);
268 }
269
270
271 for (final FileConfig fileConfig : fileConfigList) {
272 final String sid = ComponentUtil.getCrawlingConfigHelper().store(sessionId, fileConfig);
273
274
275 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
276 crawler.setSessionId(sid);
277 sessionIdList.add(sid);
278
279 final String pathsStr = fileConfig.getPaths();
280 if (StringUtil.isBlank(pathsStr)) {
281 logger.warn("[{}] No target uris. Skipped.", fileConfig.getName());
282 break;
283 }
284
285 final int intervalTime =
286 fileConfig.getIntervalTime() != null ? fileConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_FS;
287 ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
288
289 final String includedPathsStr = fileConfig.getIncludedPaths() != null ? fileConfig.getIncludedPaths() : StringUtil.EMPTY;
290 final String excludedPathsStr = fileConfig.getExcludedPaths() != null ? fileConfig.getExcludedPaths() : StringUtil.EMPTY;
291
292
293 final CrawlerContext crawlerContext = crawler.getCrawlerContext();
294 final int numOfThread =
295 fileConfig.getNumOfThread() != null ? fileConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_FS;
296 crawlerContext.setNumOfThread(numOfThread);
297
298
299 final int depth = fileConfig.getDepth() != null ? fileConfig.getDepth() : -1;
300 crawlerContext.setMaxDepth(depth);
301
302
303 final long maxCount = fileConfig.getMaxAccessCount() != null ? fileConfig.getMaxAccessCount() : maxAccessCount;
304 crawlerContext.setMaxAccessCount(maxCount);
305
306 fileConfig.initializeClientFactory(() -> crawler.getClientFactory());
307 final Map<String, String> configParamMap = fileConfig.getConfigParameterMap(ConfigName.CONFIG);
308
309 if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
310 deleteCrawlData(sid);
311 } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
312 final OpenSearchUrlFilterService urlFilterService = ComponentUtil.getComponent(OpenSearchUrlFilterService.class);
313 try {
314 urlFilterService.delete(sid);
315 } catch (final Exception e) {
316 logger.warn("Failed to delete UrlFilter: sessionId={}", sid);
317 }
318 }
319
320
321 split(pathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
322 if (!urlValue.startsWith("#")) {
323 final String u;
324 if (!protocolHelper.isValidFileProtocol(urlValue)) {
325 if (urlValue.startsWith("/")) {
326 u = "file:" + urlValue;
327 } else {
328 u = "file:/" + urlValue;
329 }
330 } else {
331 u = urlValue;
332 }
333 crawler.addUrl(u);
334 if (logger.isInfoEnabled()) {
335 logger.info("Target Path: {}", u);
336 }
337 }
338 }));
339
340
341 final AtomicBoolean urlEncodeDisabled = new AtomicBoolean(false);
342 split(includedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
343 if (!line.startsWith("#")) {
344 final String urlValue;
345 if (urlEncodeDisabled.get()) {
346 urlValue = line;
347 urlEncodeDisabled.set(false);
348 } else {
349 urlValue = systemHelper.encodeUrlFilter(line);
350 }
351 crawler.addIncludeFilter(urlValue);
352 if (logger.isInfoEnabled()) {
353 logger.info("Included Path: {}", urlValue);
354 }
355 } else if (line.startsWith(DISABLE_URL_ENCODE)) {
356 urlEncodeDisabled.set(true);
357 }
358 }));
359
360
361 urlEncodeDisabled.set(false);
362 split(excludedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
363 if (!line.startsWith("#")) {
364 final String urlValue;
365 if (urlEncodeDisabled.get()) {
366 urlValue = line;
367 urlEncodeDisabled.set(false);
368 } else {
369 urlValue = systemHelper.encodeUrlFilter(line);
370 }
371 crawler.addExcludeFilter(urlValue);
372 if (logger.isInfoEnabled()) {
373 logger.info("Excluded Path: {}", urlValue);
374 }
375 } else if (line.startsWith(DISABLE_URL_ENCODE)) {
376 urlEncodeDisabled.set(true);
377 }
378 }));
379
380
381 final List<String> excludedUrlList = ComponentUtil.getCrawlingConfigHelper().getExcludedUrlList(fileConfig.getConfigId());
382 if (excludedUrlList != null) {
383 excludedUrlList.stream().filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(u -> {
384 final String urlValue = Pattern.quote(u);
385 crawler.addExcludeFilter(urlValue);
386 if (logger.isInfoEnabled()) {
387 logger.info("Excluded Path from failures: {}", urlValue);
388 }
389 });
390 }
391
392 if (logger.isDebugEnabled()) {
393 logger.debug("Crawling {}", pathsStr);
394 }
395
396 crawler.setBackground(true);
397 crawler.setThreadPriority(crawlerPriority);
398
399 crawlerList.add(crawler);
400 crawlerStatusList.add(Constants.READY);
401 }
402
403
404 final IndexUpdater indexUpdater = ComponentUtil.getIndexUpdater();
405 indexUpdater.setName("IndexUpdater");
406 indexUpdater.setPriority(indexUpdaterPriority);
407 indexUpdater.setSessionIdList(sessionIdList);
408 indexUpdater.setDaemon(true);
409 indexUpdater.setCrawlerList(crawlerList);
410 getAvailableBoostDocumentRuleList().forEach(rule -> {
411 indexUpdater.addDocBoostMatcher(new org.codelibs.fess.indexer.DocBoostMatcher(rule));
412 });
413 indexUpdater.start();
414
415 int startedCrawlerNum = 0;
416 int activeCrawlerNum = 0;
417 try {
418 while (startedCrawlerNum < crawlerList.size()) {
419
420 if (systemHelper.isForceStop()) {
421 for (final Crawler crawler : crawlerList) {
422 crawler.stop();
423 }
424 break;
425 }
426
427 if (activeCrawlerNum < multiprocessCrawlingCount) {
428
429 crawlerList.get(startedCrawlerNum).execute();
430 crawlerStatusList.set(startedCrawlerNum, Constants.RUNNING);
431 startedCrawlerNum++;
432 activeCrawlerNum++;
433 ThreadUtil.sleep(crawlingExecutionInterval);
434 continue;
435 }
436
437
438 for (int i = 0; i < startedCrawlerNum; i++) {
439 if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE
440 && Constants.RUNNING.equals(crawlerStatusList.get(i))) {
441 crawlerList.get(i).awaitTermination();
442 crawlerStatusList.set(i, Constants.DONE);
443 final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
444 indexUpdater.addFinishedSessionId(sid);
445 activeCrawlerNum--;
446 }
447 }
448 ThreadUtil.sleep(crawlingExecutionInterval);
449 }
450
451 boolean finishedAll = false;
452 while (!finishedAll) {
453 finishedAll = true;
454 for (int i = 0; i < crawlerList.size(); i++) {
455 final Crawler crawler = crawlerList.get(i);
456 crawler.awaitTermination(crawlingExecutionInterval);
457 if (crawler.getCrawlerContext().getStatus() == CrawlerStatus.DONE && !Constants.DONE.equals(crawlerStatusList.get(i))) {
458 crawlerStatusList.set(i, Constants.DONE);
459 final String sid = crawler.getCrawlerContext().getSessionId();
460 indexUpdater.addFinishedSessionId(sid);
461 try {
462 crawler.close();
463 } catch (final Exception e) {
464 logger.warn("Failed to close the crawler.", e);
465 }
466 }
467 if (!Constants.DONE.equals(crawlerStatusList.get(i))) {
468 finishedAll = false;
469 }
470 }
471 }
472 } finally {
473 crawlerList.forEach(crawler -> {
474 try {
475 crawler.close();
476 } catch (final Exception e) {
477 logger.warn("Failed to close the crawler.", e);
478 }
479 });
480 }
481 crawlerList.clear();
482 crawlerStatusList.clear();
483
484
485 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
486
487 final long execTime = systemHelper.getCurrentTimeAsLong() - startTime;
488 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_CRAWLING_EXEC_TIME, Long.toString(execTime));
489 if (logger.isInfoEnabled()) {
490 logger.info("[EXEC TIME] crawling time: {}ms", execTime);
491 }
492
493 indexUpdater.setFinishCrawling(true);
494 try {
495 indexUpdater.join();
496 } catch (final InterruptedException e) {
497 logger.warn("Interrupted index update.", e);
498 }
499
500 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_EXEC_TIME, Long.toString(indexUpdater.getExecuteTime()));
501 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_SIZE, Long.toString(indexUpdater.getDocumentSize()));
502
503 if (systemHelper.isForceStop()) {
504 return;
505 }
506
507 for (final String sid : sessionIdList) {
508
509 ComponentUtil.getCrawlingConfigHelper().remove(sid);
510 deleteCrawlData(sid);
511 }
512 }
513
514
515
516
517
518
519 protected List<BoostDocumentRule> getAvailableBoostDocumentRuleList() {
520 return ComponentUtil.getComponent(BoostDocumentRuleBhv.class).selectList(cb -> {
521 cb.query().matchAll();
522 cb.query().addOrderBy_SortOrder_Asc();
523 cb.fetchFirst(ComponentUtil.getFessConfig().getPageDocboostMaxFetchSizeAsInteger());
524 });
525 }
526
527
528
529
530
531
532 protected void deleteCrawlData(final String sid) {
533 final OpenSearchUrlFilterService urlFilterService = ComponentUtil.getComponent(OpenSearchUrlFilterService.class);
534 final OpenSearchUrlQueueService urlQueueService = ComponentUtil.getComponent(OpenSearchUrlQueueService.class);
535 final OpenSearchDataService dataService = ComponentUtil.getComponent(OpenSearchDataService.class);
536
537 try {
538
539 urlFilterService.delete(sid);
540 } catch (final Exception e) {
541 logger.warn("Failed to delete UrlFilter: sessionId={}", sid, e);
542 }
543
544 try {
545
546 urlQueueService.clearCache();
547 urlQueueService.delete(sid);
548 } catch (final Exception e) {
549 logger.warn("Failed to delete UrlQueue: sessionId={}", sid, e);
550 }
551
552 try {
553
554 dataService.delete(sid);
555 } catch (final Exception e) {
556 logger.warn("Failed to delete AccessResult: sessionId={}", sid, e);
557 }
558 }
559
560
561
562
563
564
565 public void setMaxAccessCount(final long maxAccessCount) {
566 this.maxAccessCount = maxAccessCount;
567 }
568
569
570
571
572
573
574 public void setCrawlingExecutionInterval(final long crawlingExecutionInterval) {
575 this.crawlingExecutionInterval = crawlingExecutionInterval;
576 }
577
578
579
580
581
582
583 public void setIndexUpdaterPriority(final int indexUpdaterPriority) {
584 this.indexUpdaterPriority = indexUpdaterPriority;
585 }
586
587
588
589
590
591
592 public void setCrawlerPriority(final int crawlerPriority) {
593 this.crawlerPriority = crawlerPriority;
594 }
595
596 }