1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.exec;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedReader;
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStreamReader;
24 import java.lang.management.ManagementFactory;
25 import java.text.SimpleDateFormat;
26 import java.util.ArrayList;
27 import java.util.Collections;
28 import java.util.Date;
29 import java.util.HashMap;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.Queue;
33 import java.util.concurrent.ConcurrentLinkedQueue;
34 import java.util.concurrent.atomic.AtomicBoolean;
35 import java.util.stream.Collectors;
36
37 import javax.annotation.Resource;
38
39 import org.apache.logging.log4j.LogManager;
40 import org.apache.logging.log4j.Logger;
41 import org.codelibs.core.CoreLibConstants;
42 import org.codelibs.core.exception.InterruptedRuntimeException;
43 import org.codelibs.core.lang.StringUtil;
44 import org.codelibs.core.lang.ThreadUtil;
45 import org.codelibs.core.misc.DynamicProperties;
46 import org.codelibs.core.timer.TimeoutManager;
47 import org.codelibs.core.timer.TimeoutTask;
48 import org.codelibs.fesen.monitor.jvm.JvmInfo;
49 import org.codelibs.fesen.monitor.os.OsProbe;
50 import org.codelibs.fesen.monitor.process.ProcessProbe;
51 import org.codelibs.fess.Constants;
52 import org.codelibs.fess.app.service.CrawlingInfoService;
53 import org.codelibs.fess.app.service.PathMappingService;
54 import org.codelibs.fess.crawler.client.FesenClient;
55 import org.codelibs.fess.es.client.SearchEngineClient;
56 import org.codelibs.fess.exception.ContainerNotAvailableException;
57 import org.codelibs.fess.helper.CrawlingInfoHelper;
58 import org.codelibs.fess.helper.DataIndexHelper;
59 import org.codelibs.fess.helper.DuplicateHostHelper;
60 import org.codelibs.fess.helper.NotificationHelper;
61 import org.codelibs.fess.helper.PathMappingHelper;
62 import org.codelibs.fess.helper.WebFsIndexHelper;
63 import org.codelibs.fess.mylasta.direction.FessConfig;
64 import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
65 import org.codelibs.fess.timer.SystemMonitorTarget;
66 import org.codelibs.fess.util.ComponentUtil;
67 import org.codelibs.fess.util.ThreadDumpUtil;
68 import org.dbflute.mail.send.hook.SMailCallbackContext;
69 import org.kohsuke.args4j.CmdLineException;
70 import org.kohsuke.args4j.CmdLineParser;
71 import org.kohsuke.args4j.Option;
72 import org.lastaflute.core.mail.Postbox;
73 import org.lastaflute.di.core.external.GenericExternalContext;
74 import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
75 import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
76
77 public class Crawler {
78
79 private static final Logger logger = LogManager.getLogger(Crawler.class);
80
81 private static final String WEB_FS_CRAWLING_PROCESS = "WebFsCrawler";
82
83 private static final String DATA_CRAWLING_PROCESS = "DataStoreCrawler";
84
85 private static AtomicBoolean running = new AtomicBoolean(false);
86
87 private static Queue<String> errors = new ConcurrentLinkedQueue<>();
88
89 @Resource
90 protected SearchEngineClient searchEngineClient;
91
92 @Resource
93 protected WebFsIndexHelper webFsIndexHelper;
94
95 @Resource
96 protected DataIndexHelper dataIndexHelper;
97
98 @Resource
99 protected PathMappingService pathMappingService;
100
101 @Resource
102 protected CrawlingInfoService crawlingInfoService;
103
104 public static void addError(final String msg) {
105 if (StringUtil.isNotBlank(msg)) {
106 errors.offer(msg);
107 }
108 }
109
110 public static class Options {
111
112 @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
113 public String sessionId;
114
115 @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
116 public String name;
117
118 @Option(name = "-w", aliases = "--webConfigIds", metaVar = "webConfigIds", usage = "Web Config IDs")
119 public String webConfigIds;
120
121 @Option(name = "-f", aliases = "--fileConfigIds", metaVar = "fileConfigIds", usage = "File Config IDs")
122 public String fileConfigIds;
123
124 @Option(name = "-d", aliases = "--dataConfigIds", metaVar = "dataConfigIds", usage = "Data Config IDs")
125 public String dataConfigIds;
126
127 @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
128 public String propertiesPath;
129
130 @Option(name = "-e", aliases = "--expires", metaVar = "expires", usage = "Expires for documents")
131 public String expires;
132
133 protected Options() {
134
135 }
136
137 protected List<String> getWebConfigIdList() {
138 if (StringUtil.isNotBlank(webConfigIds)) {
139 final String[] values = webConfigIds.split(",");
140 return createConfigIdList(values);
141 }
142 return null;
143 }
144
145 protected List<String> getFileConfigIdList() {
146 if (StringUtil.isNotBlank(fileConfigIds)) {
147 final String[] values = fileConfigIds.split(",");
148 return createConfigIdList(values);
149 }
150 return null;
151 }
152
153 protected List<String> getDataConfigIdList() {
154 if (StringUtil.isNotBlank(dataConfigIds)) {
155 final String[] values = dataConfigIds.split(",");
156 return createConfigIdList(values);
157 }
158 return null;
159 }
160
161 private static List<String> createConfigIdList(final String[] values) {
162 final List<String> idList = new ArrayList<>();
163 Collections.addAll(idList, values);
164 return idList;
165 }
166
167 @Override
168 public String toString() {
169 return "Options [sessionId=" + sessionId + ", name=" + name + ", webConfigIds=" + webConfigIds + ", fileConfigIds="
170 + fileConfigIds + ", dataConfigIds=" + dataConfigIds + ", propertiesPath=" + propertiesPath + ", expires=" + expires
171 + "]";
172 }
173
174 }
175
176 static void initializeProbes() {
177
178 ProcessProbe.getInstance();
179 OsProbe.getInstance();
180 JvmInfo.jvmInfo();
181 }
182
183 public static void main(final String[] args) {
184 final Options options = new Options();
185
186 final CmdLineParser parser = new CmdLineParser(options);
187 try {
188 parser.parseArgument(args);
189 } catch (final CmdLineException e) {
190 System.err.println(e.getMessage());
191 System.err.println("java " + Crawler.class.getCanonicalName() + " [options...] arguments...");
192 parser.printUsage(System.err);
193 return;
194 }
195
196 if (logger.isDebugEnabled()) {
197 try {
198 ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
199 System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: {}={}", e.getKey(), e.getValue()));
200 System.getenv().entrySet().forEach(e -> logger.debug("Env: {}={}", e.getKey(), e.getValue()));
201 logger.debug("Option: {}", options);
202 } catch (final Exception e) {
203
204 }
205 }
206
207 initializeProbes();
208
209 final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
210 if (StringUtil.isNotBlank(httpAddress)) {
211 System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
212 }
213
214 TimeoutTask systemMonitorTask = null;
215 Thread commandThread = null;
216 int exitCode;
217 try {
218 running.set(true);
219 SingletonLaContainerFactory.setConfigPath("app.xml");
220 SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
221 SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
222 SingletonLaContainerFactory.init();
223
224 final Thread shutdownCallback = new Thread("ShutdownHook") {
225 @Override
226 public void run() {
227 destroyContainer();
228 }
229
230 };
231 Runtime.getRuntime().addShutdownHook(shutdownCallback);
232
233 commandThread = new Thread(() -> {
234 try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
235 String command;
236 while (true) {
237 try {
238 while (!reader.ready()) {
239 ThreadUtil.sleep(1000L);
240 }
241 command = reader.readLine().trim();
242 if (logger.isDebugEnabled()) {
243 logger.debug("Process command: {}", command);
244 }
245 if (Constants.CRAWLER_PROCESS_COMMAND_THREAD_DUMP.equals(command)) {
246 ThreadDumpUtil.printThreadDump();
247 } else {
248 logger.warn("Unknown process command: {}", command);
249 }
250 if (Thread.interrupted()) {
251 return;
252 }
253 } catch (final InterruptedRuntimeException e) {
254 return;
255 }
256 }
257 } catch (final IOException e) {
258 logger.debug("I/O exception.", e);
259 }
260 }, "ProcessCommand");
261 commandThread.start();
262
263 systemMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
264 ComponentUtil.getFessConfig().getCrawlerSystemMonitorIntervalAsInteger(), true);
265
266 exitCode = process(options);
267 } catch (final ContainerNotAvailableException e) {
268 if (logger.isDebugEnabled()) {
269 logger.debug("Crawler is stopped.", e);
270 } else if (logger.isInfoEnabled()) {
271 logger.info("Crawler is stopped.");
272 }
273 exitCode = Constants.EXIT_FAIL;
274 } catch (final Throwable t) {
275 logger.error("Crawler does not work correctly.", t);
276 exitCode = Constants.EXIT_FAIL;
277 } finally {
278 if (commandThread != null && commandThread.isAlive()) {
279 commandThread.interrupt();
280 }
281 if (systemMonitorTask != null) {
282 systemMonitorTask.cancel();
283 }
284 destroyContainer();
285 }
286
287 if (exitCode != Constants.EXIT_OK) {
288 System.exit(exitCode);
289 }
290 }
291
292 private static void destroyContainer() {
293 if (running.getAndSet(false)) {
294 TimeoutManager.getInstance().stop();
295 if (logger.isDebugEnabled()) {
296 logger.debug("Destroying LaContainer...");
297 }
298 SingletonLaContainerFactory.destroy();
299 logger.info("Destroyed LaContainer.");
300 }
301 }
302
303 private static int process(final Options options) {
304 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
305
306 if (StringUtil.isBlank(options.sessionId)) {
307
308 final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
309 options.sessionId = sdf.format(new Date());
310 } else {
311 options.sessionId = options.sessionId.replace('-', '_');
312 }
313
314 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
315 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
316
317 if (StringUtil.isNotBlank(options.propertiesPath)) {
318 systemProperties.reload(options.propertiesPath);
319 } else {
320 try {
321 final File propFile = ComponentUtil.getSystemHelper().createTempFile("crawler_", ".properties");
322 if (propFile.delete() && logger.isDebugEnabled()) {
323 logger.debug("Deleted a temp file: {}", propFile.getAbsolutePath());
324 }
325 systemProperties.reload(propFile.getAbsolutePath());
326 propFile.deleteOnExit();
327 } catch (final Exception e) {
328 logger.warn("Failed to create system properties file.", e);
329 }
330 }
331
332 try {
333 crawlingInfoHelper.store(options.sessionId, true);
334 final String dayForCleanupStr;
335 int dayForCleanup = -1;
336 if (StringUtil.isNotBlank(options.expires)) {
337 dayForCleanupStr = options.expires;
338 try {
339 dayForCleanup = Integer.parseInt(dayForCleanupStr);
340 } catch (final NumberFormatException e) {}
341 } else {
342 dayForCleanup = ComponentUtil.getFessConfig().getDayForCleanup();
343 }
344 crawlingInfoHelper.updateParams(options.sessionId, options.name, dayForCleanup);
345 } catch (final Exception e) {
346 logger.warn("Failed to store crawling information.", e);
347 }
348
349 try {
350 return crawler.doCrawl(options);
351 } finally {
352 try {
353 crawlingInfoHelper.store(options.sessionId, false);
354 } catch (final Exception e) {
355 logger.warn("Failed to store crawling information.", e);
356 }
357
358 final Map<String, String> infoMap = crawlingInfoHelper.getInfoMap(options.sessionId);
359
360 final StringBuilder buf = new StringBuilder(500);
361 for (final Map.Entry<String, String> entry : infoMap.entrySet()) {
362 if (buf.length() != 0) {
363 buf.append(',');
364 }
365 buf.append(entry.getKey()).append('=').append(entry.getValue());
366 }
367 logger.info("[CRAWL INFO] {}", buf.toString());
368
369
370 try {
371 crawler.sendMail(infoMap);
372 } catch (final Exception e) {
373 logger.warn("Failed to send a mail.", e);
374 }
375
376 }
377 }
378
379 protected void sendMail(final Map<String, String> infoMap) {
380 final FessConfig fessConfig = ComponentUtil.getFessConfig();
381 if (fessConfig.hasNotification()) {
382 final Map<String, String> dataMap = new HashMap<>();
383 for (final Map.Entry<String, String> entry : infoMap.entrySet()) {
384 dataMap.put(StringUtil.decapitalize(entry.getKey()), entry.getValue());
385 }
386
387 String hostname = fessConfig.getMailHostname();
388 if (StringUtil.isBlank(hostname)) {
389 hostname = ComponentUtil.getSystemHelper().getHostname();
390 }
391 dataMap.put("hostname", hostname);
392
393 logger.debug("\ninfoMap: {}\ndataMap: {}", infoMap, dataMap);
394
395 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
396 final String toStrs = fessConfig.getNotificationTo();
397 final Postbox postbox = ComponentUtil.getComponent(Postbox.class);
398 try {
399 final String[] toAddresses;
400 if (StringUtil.isNotBlank(toStrs)) {
401 toAddresses = toStrs.split(",");
402 } else {
403 toAddresses = StringUtil.EMPTY_STRINGS;
404 }
405 final NotificationHelper notificationHelper = ComponentUtil.getNotificationHelper();
406 SMailCallbackContext.setPreparedMessageHookOnThread(notificationHelper::send);
407 CrawlerPostcard.droppedInto(postbox, postcard -> {
408 postcard.setFrom(fessConfig.getMailFromAddress(), fessConfig.getMailFromName());
409 postcard.addReplyTo(fessConfig.getMailReturnPath());
410 if (toAddresses.length > 0) {
411 stream(toAddresses).of(stream -> stream.map(String::trim).forEach(address -> {
412 postcard.addTo(address);
413 }));
414 } else {
415 postcard.addTo(fessConfig.getMailFromAddress());
416 postcard.dryrun();
417 }
418 postcard.setCrawlerEndTime(getValueFromMap(dataMap, "crawlerEndTime", StringUtil.EMPTY));
419 postcard.setCrawlerExecTime(getValueFromMap(dataMap, "crawlerExecTime", "0"));
420 postcard.setCrawlerStartTime(getValueFromMap(dataMap, "crawlerStartTime", StringUtil.EMPTY));
421 postcard.setDataCrawlEndTime(getValueFromMap(dataMap, "dataCrawlEndTime", StringUtil.EMPTY));
422 postcard.setDataCrawlExecTime(getValueFromMap(dataMap, "dataCrawlExecTime", "0"));
423 postcard.setDataCrawlStartTime(getValueFromMap(dataMap, "dataCrawlStartTime", StringUtil.EMPTY));
424 postcard.setDataIndexSize(getValueFromMap(dataMap, "dataIndexSize", "0"));
425 postcard.setDataIndexExecTime(getValueFromMap(dataMap, "dataIndexExecTime", "0"));
426 postcard.setHostname(getValueFromMap(dataMap, "hostname", StringUtil.EMPTY));
427 postcard.setWebFsCrawlEndTime(getValueFromMap(dataMap, "webFsCrawlEndTime", StringUtil.EMPTY));
428 postcard.setWebFsCrawlExecTime(getValueFromMap(dataMap, "webFsCrawlExecTime", "0"));
429 postcard.setWebFsCrawlStartTime(getValueFromMap(dataMap, "webFsCrawlStartTime", StringUtil.EMPTY));
430 postcard.setWebFsIndexExecTime(getValueFromMap(dataMap, "webFsIndexExecTime", "0"));
431 postcard.setWebFsIndexSize(getValueFromMap(dataMap, "webFsIndexSize", "0"));
432 if (Constants.TRUE.equalsIgnoreCase(infoMap.get(Constants.CRAWLER_STATUS))) {
433 postcard.setStatus(Constants.OK);
434 } else {
435 postcard.setStatus(Constants.FAIL);
436 }
437 postcard.setJobname(systemProperties.getProperty("job.runtime.name", StringUtil.EMPTY));
438 });
439 } finally {
440 SMailCallbackContext.clearPreparedMessageHookOnThread();
441 }
442 }
443 }
444
445 private String getValueFromMap(final Map<String, String> dataMap, final String key, final String defaultValue) {
446 final String value = dataMap.get(key);
447 if (StringUtil.isBlank(value)) {
448 return defaultValue;
449 }
450 return value;
451 }
452
453 public int doCrawl(final Options options) {
454 if (logger.isInfoEnabled()) {
455 logger.info("Starting Crawler..");
456 }
457
458 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
459
460 final long totalTime = System.currentTimeMillis();
461
462 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
463
464 try {
465 writeTimeToSessionInfo(crawlingInfoHelper, Constants.CRAWLER_START_TIME);
466
467
468 final List<String> ptList = new ArrayList<>();
469 ptList.add(Constants.PROCESS_TYPE_CRAWLING);
470 ptList.add(Constants.PROCESS_TYPE_BOTH);
471 pathMappingHelper.setPathMappingList(options.sessionId, pathMappingService.getPathMappingList(ptList));
472
473
474 try {
475 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
476 duplicateHostHelper.init();
477 } catch (final Exception e) {
478 logger.warn("Could not initialize duplicateHostHelper.", e);
479 }
480
481
482 crawlingInfoService.deleteSessionIdsBefore(options.sessionId, options.name,
483 ComponentUtil.getSystemHelper().getCurrentTimeAsLong());
484
485 final List<String> webConfigIdList = options.getWebConfigIdList();
486 final List<String> fileConfigIdList = options.getFileConfigIdList();
487 final List<String> dataConfigIdList = options.getDataConfigIdList();
488 final boolean runAll = webConfigIdList == null && fileConfigIdList == null && dataConfigIdList == null;
489
490 Thread webFsCrawlerThread = null;
491 Thread dataCrawlerThread = null;
492
493 if (runAll || webConfigIdList != null || fileConfigIdList != null) {
494 webFsCrawlerThread = new Thread((Runnable) () -> {
495
496 writeTimeToSessionInfo(crawlingInfoHelper, Constants.WEB_FS_CRAWLER_START_TIME);
497 webFsIndexHelper.crawl(options.sessionId, webConfigIdList, fileConfigIdList);
498 writeTimeToSessionInfo(crawlingInfoHelper, Constants.WEB_FS_CRAWLER_END_TIME);
499 }, WEB_FS_CRAWLING_PROCESS);
500 webFsCrawlerThread.start();
501 }
502
503 if (runAll || dataConfigIdList != null) {
504 dataCrawlerThread = new Thread((Runnable) () -> {
505
506 writeTimeToSessionInfo(crawlingInfoHelper, Constants.DATA_CRAWLER_START_TIME);
507 dataIndexHelper.crawl(options.sessionId, dataConfigIdList);
508 writeTimeToSessionInfo(crawlingInfoHelper, Constants.DATA_CRAWLER_END_TIME);
509 }, DATA_CRAWLING_PROCESS);
510 dataCrawlerThread.start();
511 }
512
513 joinCrawlerThread(webFsCrawlerThread);
514 joinCrawlerThread(dataCrawlerThread);
515
516 if (logger.isInfoEnabled()) {
517 logger.info("Finished Crawler");
518 }
519
520 return Constants.EXIT_OK;
521 } catch (final Throwable t) {
522 logger.warn("An exception occurs on the crawl task.", t);
523 return Constants.EXIT_FAIL;
524 } finally {
525 pathMappingHelper.removePathMappingList(options.sessionId);
526 crawlingInfoHelper.putToInfoMap(Constants.CRAWLER_STATUS, errors.isEmpty() ? Constants.T.toString() : Constants.F.toString());
527 if (!errors.isEmpty()) {
528 crawlingInfoHelper.putToInfoMap(Constants.CRAWLER_ERRORS,
529 errors.stream().map(s -> s.replace(" ", StringUtil.EMPTY)).collect(Collectors.joining(" ")));
530 }
531 writeTimeToSessionInfo(crawlingInfoHelper, Constants.CRAWLER_END_TIME);
532 crawlingInfoHelper.putToInfoMap(Constants.CRAWLER_EXEC_TIME, Long.toString(System.currentTimeMillis() - totalTime));
533
534 }
535 }
536
537 protected void writeTimeToSessionInfo(final CrawlingInfoHelper crawlingInfoHelper, final String key) {
538 if (crawlingInfoHelper != null) {
539 final SimpleDateFormat dateFormat = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
540 crawlingInfoHelper.putToInfoMap(key, dateFormat.format(new Date()));
541 }
542 }
543
544 private void joinCrawlerThread(final Thread crawlerThread) {
545 if (crawlerThread != null) {
546 try {
547 crawlerThread.join();
548 } catch (final Exception e) {
549 logger.info("Interrupted a crawling process: {}", crawlerThread.getName());
550 }
551 }
552 }
553 }