1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.util.ArrayList;
21 import java.util.Collections;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.concurrent.atomic.AtomicBoolean;
25 import java.util.regex.Pattern;
26
27 import org.apache.logging.log4j.LogManager;
28 import org.apache.logging.log4j.Logger;
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.core.lang.ThreadUtil;
31 import org.codelibs.fess.Constants;
32 import org.codelibs.fess.crawler.Crawler;
33 import org.codelibs.fess.crawler.CrawlerContext;
34 import org.codelibs.fess.crawler.CrawlerStatus;
35 import org.codelibs.fess.crawler.interval.FessIntervalController;
36 import org.codelibs.fess.crawler.service.impl.EsDataService;
37 import org.codelibs.fess.crawler.service.impl.EsUrlFilterService;
38 import org.codelibs.fess.crawler.service.impl.EsUrlQueueService;
39 import org.codelibs.fess.es.config.exbhv.BoostDocumentRuleBhv;
40 import org.codelibs.fess.es.config.exentity.BoostDocumentRule;
41 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
42 import org.codelibs.fess.es.config.exentity.CrawlingConfig.Param.Config;
43 import org.codelibs.fess.es.config.exentity.FileConfig;
44 import org.codelibs.fess.es.config.exentity.WebConfig;
45 import org.codelibs.fess.indexer.IndexUpdater;
46 import org.codelibs.fess.mylasta.direction.FessConfig;
47 import org.codelibs.fess.util.ComponentUtil;
48
49 public class WebFsIndexHelper {
50
51 private static final Logger logger = LogManager.getLogger(WebFsIndexHelper.class);
52
53 protected long maxAccessCount = Long.MAX_VALUE;
54
55 protected long crawlingExecutionInterval = Constants.DEFAULT_CRAWLING_EXECUTION_INTERVAL;
56
57 protected int indexUpdaterPriority = Thread.MAX_PRIORITY;
58
59 protected int crawlerPriority = Thread.NORM_PRIORITY;
60
61 protected final List<Crawler> crawlerList = Collections.synchronizedList(new ArrayList<Crawler>());
62
63 public void crawl(final String sessionId, final List<String> webConfigIdList, final List<String> fileConfigIdList) {
64 final boolean runAll = webConfigIdList == null && fileConfigIdList == null;
65 final List<WebConfig> webConfigList;
66 if (runAll || webConfigIdList != null) {
67 webConfigList = ComponentUtil.getCrawlingConfigHelper().getWebConfigListByIds(webConfigIdList);
68 } else {
69 webConfigList = Collections.emptyList();
70 }
71 final List<FileConfig> fileConfigList;
72 if (runAll || fileConfigIdList != null) {
73 fileConfigList = ComponentUtil.getCrawlingConfigHelper().getFileConfigListByIds(fileConfigIdList);
74 } else {
75 fileConfigList = Collections.emptyList();
76 }
77
78 if (webConfigList.isEmpty() && fileConfigList.isEmpty()) {
79
80 if (logger.isInfoEnabled()) {
81 logger.info("No crawling target urls.");
82 }
83 return;
84 }
85
86 doCrawl(sessionId, webConfigList, fileConfigList);
87 }
88
89 protected void doCrawl(final String sessionId, final List<WebConfig> webConfigList, final List<FileConfig> fileConfigList) {
90 final int multiprocessCrawlingCount = ComponentUtil.getFessConfig().getCrawlingThreadCount();
91
92 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
93 final FessConfig fessConfig = ComponentUtil.getFessConfig();
94
95 final long startTime = System.currentTimeMillis();
96
97 final List<String> sessionIdList = new ArrayList<>();
98 crawlerList.clear();
99 final List<String> crawlerStatusList = new ArrayList<>();
100
101 for (final WebConfig webConfig : webConfigList) {
102 final String sid = ComponentUtil.getCrawlingConfigHelper().store(sessionId, webConfig);
103
104
105 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
106 crawler.setSessionId(sid);
107 sessionIdList.add(sid);
108
109 final String urlsStr = webConfig.getUrls();
110 if (StringUtil.isBlank(urlsStr)) {
111 logger.warn("No target urls. Skipped");
112 break;
113 }
114
115
116 final int intervalTime =
117 webConfig.getIntervalTime() != null ? webConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_WEB;
118 ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
119
120 final String includedUrlsStr = webConfig.getIncludedUrls() != null ? webConfig.getIncludedUrls() : StringUtil.EMPTY;
121 final String excludedUrlsStr = webConfig.getExcludedUrls() != null ? webConfig.getExcludedUrls() : StringUtil.EMPTY;
122
123
124 final CrawlerContext crawlerContext = crawler.getCrawlerContext();
125 final int numOfThread =
126 webConfig.getNumOfThread() != null ? webConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_WEB;
127 crawlerContext.setNumOfThread(numOfThread);
128
129
130 final int depth = webConfig.getDepth() != null ? webConfig.getDepth() : -1;
131 crawlerContext.setMaxDepth(depth);
132
133
134 final long maxCount = webConfig.getMaxAccessCount() != null ? webConfig.getMaxAccessCount() : maxAccessCount;
135 crawlerContext.setMaxAccessCount(maxCount);
136
137 webConfig.initializeClientFactory(() -> crawler.getClientFactory());
138 final Map<String, String> configParamMap = webConfig.getConfigParameterMap(ConfigName.CONFIG);
139
140 if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
141 deleteCrawlData(sid);
142 } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
143 final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
144 try {
145 urlFilterService.delete(sid);
146 } catch (final Exception e) {
147 logger.warn("Failed to delete url filters for {}", sid);
148 }
149 }
150
151 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
152
153
154 split(urlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
155 if (!urlValue.startsWith("#") && fessConfig.isValidCrawlerWebProtocol(urlValue)) {
156 final String u = duplicateHostHelper.convert(urlValue);
157 crawler.addUrl(u);
158 if (logger.isInfoEnabled()) {
159 logger.info("Target URL: {}", u);
160 }
161 }
162 }));
163
164
165 split(includedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
166 if (!urlValue.startsWith("#")) {
167 crawler.addIncludeFilter(urlValue);
168 if (logger.isInfoEnabled()) {
169 logger.info("Included URL: {}", urlValue);
170 }
171 }
172 }));
173
174
175 split(excludedUrlsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(urlValue -> {
176 if (!urlValue.startsWith("#")) {
177 crawler.addExcludeFilter(urlValue);
178 if (logger.isInfoEnabled()) {
179 logger.info("Excluded URL: {}", urlValue);
180 }
181 }
182 }));
183
184
185 final List<String> excludedUrlList = ComponentUtil.getCrawlingConfigHelper().getExcludedUrlList(webConfig.getConfigId());
186 if (excludedUrlList != null) {
187 excludedUrlList.stream().filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(u -> {
188 final String urlValue = Pattern.quote(u);
189 crawler.addExcludeFilter(urlValue);
190 if (logger.isInfoEnabled()) {
191 logger.info("Excluded URL from failures: {}", urlValue);
192 }
193 });
194 }
195
196 if (logger.isDebugEnabled()) {
197 logger.debug("Crawling {}", urlsStr);
198 }
199
200 crawler.setBackground(true);
201 crawler.setThreadPriority(crawlerPriority);
202
203 crawlerList.add(crawler);
204 crawlerStatusList.add(Constants.READY);
205 }
206
207
208 for (final FileConfig fileConfig : fileConfigList) {
209 final String sid = ComponentUtil.getCrawlingConfigHelper().store(sessionId, fileConfig);
210
211
212 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
213 crawler.setSessionId(sid);
214 sessionIdList.add(sid);
215
216 final String pathsStr = fileConfig.getPaths();
217 if (StringUtil.isBlank(pathsStr)) {
218 logger.warn("No target uris. Skipped");
219 break;
220 }
221
222 final int intervalTime =
223 fileConfig.getIntervalTime() != null ? fileConfig.getIntervalTime() : Constants.DEFAULT_INTERVAL_TIME_FOR_FS;
224 ((FessIntervalController) crawler.getIntervalController()).setDelayMillisForWaitingNewUrl(intervalTime);
225
226 final String includedPathsStr = fileConfig.getIncludedPaths() != null ? fileConfig.getIncludedPaths() : StringUtil.EMPTY;
227 final String excludedPathsStr = fileConfig.getExcludedPaths() != null ? fileConfig.getExcludedPaths() : StringUtil.EMPTY;
228
229
230 final CrawlerContext crawlerContext = crawler.getCrawlerContext();
231 final int numOfThread =
232 fileConfig.getNumOfThread() != null ? fileConfig.getNumOfThread() : Constants.DEFAULT_NUM_OF_THREAD_FOR_FS;
233 crawlerContext.setNumOfThread(numOfThread);
234
235
236 final int depth = fileConfig.getDepth() != null ? fileConfig.getDepth() : -1;
237 crawlerContext.setMaxDepth(depth);
238
239
240 final long maxCount = fileConfig.getMaxAccessCount() != null ? fileConfig.getMaxAccessCount() : maxAccessCount;
241 crawlerContext.setMaxAccessCount(maxCount);
242
243 fileConfig.initializeClientFactory(() -> crawler.getClientFactory());
244 final Map<String, String> configParamMap = fileConfig.getConfigParameterMap(ConfigName.CONFIG);
245
246 if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_ALL))) {
247 deleteCrawlData(sid);
248 } else if (Constants.TRUE.equalsIgnoreCase(configParamMap.get(Config.CLEANUP_URL_FILTERS))) {
249 final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
250 try {
251 urlFilterService.delete(sid);
252 } catch (final Exception e) {
253 logger.warn("Failed to delete url filters for {}", sid);
254 }
255 }
256
257
258 split(pathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(urlValue -> {
259 if (!urlValue.startsWith("#")) {
260 final String u;
261 if (!fessConfig.isValidCrawlerFileProtocol(urlValue)) {
262 if (urlValue.startsWith("/")) {
263 u = "file:" + urlValue;
264 } else {
265 u = "file:/" + urlValue;
266 }
267 } else {
268 u = urlValue;
269 }
270 crawler.addUrl(u);
271 if (logger.isInfoEnabled()) {
272 logger.info("Target Path: {}", u);
273 }
274 }
275 }));
276
277
278 final AtomicBoolean urlEncodeDisabled = new AtomicBoolean(false);
279 split(includedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
280 if (!line.startsWith("#")) {
281 final String urlValue;
282 if (urlEncodeDisabled.get()) {
283 urlValue = line;
284 urlEncodeDisabled.set(false);
285 } else {
286 urlValue = systemHelper.encodeUrlFilter(line);
287 }
288 crawler.addIncludeFilter(urlValue);
289 if (logger.isInfoEnabled()) {
290 logger.info("Included Path: {}", urlValue);
291 }
292 } else if (line.startsWith("#DISABLE_URL_ENCODE")) {
293 urlEncodeDisabled.set(true);
294 }
295 }));
296
297
298 urlEncodeDisabled.set(false);
299 split(excludedPathsStr, "[\r\n]").of(stream -> stream.filter(StringUtil::isNotBlank).map(String::trim).forEach(line -> {
300 if (!line.startsWith("#")) {
301 final String urlValue;
302 if (urlEncodeDisabled.get()) {
303 urlValue = line;
304 urlEncodeDisabled.set(false);
305 } else {
306 urlValue = systemHelper.encodeUrlFilter(line);
307 }
308 crawler.addExcludeFilter(urlValue);
309 if (logger.isInfoEnabled()) {
310 logger.info("Excluded Path: {}", urlValue);
311 }
312 } else if (line.startsWith("#DISABLE_URL_ENCODE")) {
313 urlEncodeDisabled.set(true);
314 }
315 }));
316
317
318 final List<String> excludedUrlList = ComponentUtil.getCrawlingConfigHelper().getExcludedUrlList(fileConfig.getConfigId());
319 if (excludedUrlList != null) {
320 excludedUrlList.stream().filter(StringUtil::isNotBlank).map(String::trim).distinct().forEach(u -> {
321 final String urlValue = Pattern.quote(u);
322 crawler.addExcludeFilter(urlValue);
323 if (logger.isInfoEnabled()) {
324 logger.info("Excluded Path from failures: {}", urlValue);
325 }
326 });
327 }
328
329 if (logger.isDebugEnabled()) {
330 logger.debug("Crawling {}", pathsStr);
331 }
332
333 crawler.setBackground(true);
334 crawler.setThreadPriority(crawlerPriority);
335
336 crawlerList.add(crawler);
337 crawlerStatusList.add(Constants.READY);
338 }
339
340
341 final IndexUpdater indexUpdater = ComponentUtil.getIndexUpdater();
342 indexUpdater.setName("IndexUpdater");
343 indexUpdater.setPriority(indexUpdaterPriority);
344 indexUpdater.setSessionIdList(sessionIdList);
345 indexUpdater.setDaemon(true);
346 indexUpdater.setCrawlerList(crawlerList);
347 getAvailableBoostDocumentRuleList().forEach(rule -> {
348 indexUpdater.addDocBoostMatcher(new org.codelibs.fess.indexer.DocBoostMatcher(rule));
349 });
350 indexUpdater.start();
351
352 int startedCrawlerNum = 0;
353 int activeCrawlerNum = 0;
354 while (startedCrawlerNum < crawlerList.size()) {
355
356 if (systemHelper.isForceStop()) {
357 for (final Crawler crawler : crawlerList) {
358 crawler.stop();
359 }
360 break;
361 }
362
363 if (activeCrawlerNum < multiprocessCrawlingCount) {
364
365 crawlerList.get(startedCrawlerNum).execute();
366 crawlerStatusList.set(startedCrawlerNum, Constants.RUNNING);
367 startedCrawlerNum++;
368 activeCrawlerNum++;
369 ThreadUtil.sleep(crawlingExecutionInterval);
370 continue;
371 }
372
373
374 for (int i = 0; i < startedCrawlerNum; i++) {
375 if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE
376 && Constants.RUNNING.equals(crawlerStatusList.get(i))) {
377 crawlerList.get(i).awaitTermination();
378 crawlerStatusList.set(i, Constants.DONE);
379 final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
380 indexUpdater.addFinishedSessionId(sid);
381 activeCrawlerNum--;
382 }
383 }
384 ThreadUtil.sleep(crawlingExecutionInterval);
385 }
386
387 boolean finishedAll = false;
388 while (!finishedAll) {
389 finishedAll = true;
390 for (int i = 0; i < crawlerList.size(); i++) {
391 crawlerList.get(i).awaitTermination(crawlingExecutionInterval);
392 if (crawlerList.get(i).getCrawlerContext().getStatus() == CrawlerStatus.DONE
393 && !Constants.DONE.equals(crawlerStatusList.get(i))) {
394 crawlerStatusList.set(i, Constants.DONE);
395 final String sid = crawlerList.get(i).getCrawlerContext().getSessionId();
396 indexUpdater.addFinishedSessionId(sid);
397 }
398 if (!Constants.DONE.equals(crawlerStatusList.get(i))) {
399 finishedAll = false;
400 }
401 }
402 }
403 crawlerList.clear();
404 crawlerStatusList.clear();
405
406
407 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
408
409 final long execTime = System.currentTimeMillis() - startTime;
410 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_CRAWLING_EXEC_TIME, Long.toString(execTime));
411 if (logger.isInfoEnabled()) {
412 logger.info("[EXEC TIME] crawling time: {}ms", execTime);
413 }
414
415 indexUpdater.setFinishCrawling(true);
416 try {
417 indexUpdater.join();
418 } catch (final InterruptedException e) {
419 logger.warn("Interrupted index update.", e);
420 }
421
422 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_EXEC_TIME, Long.toString(indexUpdater.getExecuteTime()));
423 crawlingInfoHelper.putToInfoMap(Constants.WEB_FS_INDEX_SIZE, Long.toString(indexUpdater.getDocumentSize()));
424
425 if (systemHelper.isForceStop()) {
426 return;
427 }
428
429 for (final String sid : sessionIdList) {
430
431 ComponentUtil.getCrawlingConfigHelper().remove(sid);
432 deleteCrawlData(sid);
433 }
434 }
435
436 protected List<BoostDocumentRule> getAvailableBoostDocumentRuleList() {
437 return ComponentUtil.getComponent(BoostDocumentRuleBhv.class).selectList(cb -> {
438 cb.query().matchAll();
439 cb.query().addOrderBy_SortOrder_Asc();
440 cb.fetchFirst(ComponentUtil.getFessConfig().getPageDocboostMaxFetchSizeAsInteger());
441 });
442 }
443
444 protected void deleteCrawlData(final String sid) {
445 final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
446 final EsUrlQueueService urlQueueService = ComponentUtil.getComponent(EsUrlQueueService.class);
447 final EsDataService dataService = ComponentUtil.getComponent(EsDataService.class);
448
449 try {
450
451 urlFilterService.delete(sid);
452 } catch (final Exception e) {
453 logger.warn("Failed to delete UrlFilter for {}", sid, e);
454 }
455
456 try {
457
458 urlQueueService.clearCache();
459 urlQueueService.delete(sid);
460 } catch (final Exception e) {
461 logger.warn("Failed to delete UrlQueue for {}", sid, e);
462 }
463
464 try {
465
466 dataService.delete(sid);
467 } catch (final Exception e) {
468 logger.warn("Failed to delete AccessResult for {}", sid, e);
469 }
470 }
471
472 public void setMaxAccessCount(final long maxAccessCount) {
473 this.maxAccessCount = maxAccessCount;
474 }
475
476 public void setCrawlingExecutionInterval(final long crawlingExecutionInterval) {
477 this.crawlingExecutionInterval = crawlingExecutionInterval;
478 }
479
480 public void setIndexUpdaterPriority(final int indexUpdaterPriority) {
481 this.indexUpdaterPriority = indexUpdaterPriority;
482 }
483
484 public void setCrawlerPriority(final int crawlerPriority) {
485 this.crawlerPriority = crawlerPriority;
486 }
487
488 }