1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.exec;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.File;
21 import java.io.IOException;
22 import java.lang.management.ManagementFactory;
23 import java.text.SimpleDateFormat;
24 import java.util.ArrayList;
25 import java.util.Date;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Queue;
30 import java.util.concurrent.ConcurrentLinkedQueue;
31 import java.util.concurrent.atomic.AtomicBoolean;
32 import java.util.stream.Collectors;
33
34 import javax.annotation.Resource;
35
36 import org.codelibs.core.CoreLibConstants;
37 import org.codelibs.core.lang.StringUtil;
38 import org.codelibs.core.misc.DynamicProperties;
39 import org.codelibs.fess.Constants;
40 import org.codelibs.fess.app.service.CrawlingInfoService;
41 import org.codelibs.fess.app.service.PathMappingService;
42 import org.codelibs.fess.crawler.client.EsClient;
43 import org.codelibs.fess.es.client.FessEsClient;
44 import org.codelibs.fess.exception.ContainerNotAvailableException;
45 import org.codelibs.fess.helper.CrawlingInfoHelper;
46 import org.codelibs.fess.helper.DataIndexHelper;
47 import org.codelibs.fess.helper.DuplicateHostHelper;
48 import org.codelibs.fess.helper.PathMappingHelper;
49 import org.codelibs.fess.helper.WebFsIndexHelper;
50 import org.codelibs.fess.mylasta.direction.FessConfig;
51 import org.codelibs.fess.mylasta.mail.CrawlerPostcard;
52 import org.codelibs.fess.util.ComponentUtil;
53 import org.kohsuke.args4j.CmdLineException;
54 import org.kohsuke.args4j.CmdLineParser;
55 import org.kohsuke.args4j.Option;
56 import org.lastaflute.core.mail.Postbox;
57 import org.lastaflute.di.core.external.GenericExternalContext;
58 import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
59 import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
60 import org.slf4j.Logger;
61 import org.slf4j.LoggerFactory;
62
63 public class Crawler {
64
65 private static final Logger logger = LoggerFactory.getLogger(Crawler.class);
66
67 private static final String WEB_FS_CRAWLING_PROCESS = "WebFsCrawler";
68
69 private static final String DATA_CRAWLING_PROCESS = "DataStoreCrawler";
70
71 private static AtomicBoolean running = new AtomicBoolean(false);
72
73 private static Queue<String> errors = new ConcurrentLinkedQueue<>();
74
75 @Resource
76 protected FessEsClient fessEsClient;
77
78 @Resource
79 protected WebFsIndexHelper webFsIndexHelper;
80
81 @Resource
82 protected DataIndexHelper dataIndexHelper;
83
84 @Resource
85 protected PathMappingService pathMappingService;
86
87 @Resource
88 protected CrawlingInfoService crawlingInfoService;
89
90 public static void addError(final String msg) {
91 if (StringUtil.isNotBlank(msg)) {
92 errors.offer(msg);
93 }
94 }
95
96 public static class Options {
97
98 @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
99 public String sessionId;
100
101 @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
102 public String name;
103
104 @Option(name = "-w", aliases = "--webConfigIds", metaVar = "webConfigIds", usage = "Web Config IDs")
105 public String webConfigIds;
106
107 @Option(name = "-f", aliases = "--fileConfigIds", metaVar = "fileConfigIds", usage = "File Config IDs")
108 public String fileConfigIds;
109
110 @Option(name = "-d", aliases = "--dataConfigIds", metaVar = "dataConfigIds", usage = "Data Config IDs")
111 public String dataConfigIds;
112
113 @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
114 public String propertiesPath;
115
116 @Option(name = "-e", aliases = "--expires", metaVar = "expires", usage = "Expires for documents")
117 public String expires;
118
119 protected Options() {
120
121 }
122
123 protected List<String> getWebConfigIdList() {
124 if (StringUtil.isNotBlank(webConfigIds)) {
125 final String[] values = webConfigIds.split(",");
126 return createConfigIdList(values);
127 }
128 return null;
129 }
130
131 protected List<String> getFileConfigIdList() {
132 if (StringUtil.isNotBlank(fileConfigIds)) {
133 final String[] values = fileConfigIds.split(",");
134 return createConfigIdList(values);
135 }
136 return null;
137 }
138
139 protected List<String> getDataConfigIdList() {
140 if (StringUtil.isNotBlank(dataConfigIds)) {
141 final String[] values = dataConfigIds.split(",");
142 return createConfigIdList(values);
143 }
144 return null;
145 }
146
147 private static List<String> createConfigIdList(final String[] values) {
148 final List<String> idList = new ArrayList<>();
149 for (final String value : values) {
150 idList.add(value);
151 }
152 return idList;
153 }
154
155 @Override
156 public String toString() {
157 return "Options [sessionId=" + sessionId + ", name=" + name + ", webConfigIds=" + webConfigIds + ", fileConfigIds="
158 + fileConfigIds + ", dataConfigIds=" + dataConfigIds + ", propertiesPath=" + propertiesPath + ", expires=" + expires
159 + "]";
160 }
161
162 }
163
164 public static void main(final String[] args) {
165 final Options options = new Options();
166
167 final CmdLineParser parser = new CmdLineParser(options);
168 try {
169 parser.parseArgument(args);
170 } catch (final CmdLineException e) {
171 System.err.println(e.getMessage());
172 System.err.println("java " + Crawler.class.getCanonicalName() + " [options...] arguments...");
173 parser.printUsage(System.err);
174 return;
175 }
176
177 if (logger.isDebugEnabled()) {
178 try {
179 ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: " + s));
180 System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: " + e.getKey() + "=" + e.getValue()));
181 System.getenv().entrySet().forEach(e -> logger.debug("Env: " + e.getKey() + "=" + e.getValue()));
182 logger.debug("Option: " + options);
183 } catch (final Exception e) {
184
185 }
186 }
187
188 final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
189 if (StringUtil.isNotBlank(transportAddresses)) {
190 System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
191 }
192 final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
193 if (StringUtil.isNotBlank(clusterName)) {
194 System.setProperty(EsClient.CLUSTER_NAME, clusterName);
195 }
196
197 int exitCode;
198 try {
199 running.set(true);
200 SingletonLaContainerFactory.setConfigPath("app.xml");
201 SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
202 SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
203 SingletonLaContainerFactory.init();
204
205 final Thread shutdownCallback = new Thread("ShutdownHook") {
206 @Override
207 public void run() {
208 destroyContainer();
209 }
210
211 };
212 Runtime.getRuntime().addShutdownHook(shutdownCallback);
213
214 exitCode = process(options);
215 } catch (final ContainerNotAvailableException e) {
216 if (logger.isDebugEnabled()) {
217 logger.debug("Crawler is stopped.", e);
218 } else if (logger.isInfoEnabled()) {
219 logger.info("Crawler is stopped.");
220 }
221 exitCode = Constants.EXIT_FAIL;
222 } catch (final Throwable t) {
223 logger.error("Crawler does not work correctly.", t);
224 exitCode = Constants.EXIT_FAIL;
225 } finally {
226 destroyContainer();
227 }
228
229 if (exitCode != Constants.EXIT_OK) {
230 System.exit(exitCode);
231 }
232 }
233
234 private static void destroyContainer() {
235 if (running.getAndSet(false)) {
236 if (logger.isDebugEnabled()) {
237 logger.debug("Destroying LaContainer...");
238 }
239 SingletonLaContainerFactory.destroy();
240 logger.info("Destroyed LaContainer.");
241 }
242 }
243
244 private static int process(final Options options) {
245 final Crawler crawler = ComponentUtil.getComponent(Crawler.class);
246
247 if (StringUtil.isBlank(options.sessionId)) {
248
249 final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
250 options.sessionId = sdf.format(new Date());
251 }
252
253 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
254 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
255
256 if (StringUtil.isNotBlank(options.propertiesPath)) {
257 systemProperties.reload(options.propertiesPath);
258 } else {
259 try {
260 final File propFile = File.createTempFile("crawler_", ".properties");
261 if (propFile.delete() && logger.isDebugEnabled()) {
262 logger.debug("Deleted a temp file: " + propFile.getAbsolutePath());
263 }
264 systemProperties.reload(propFile.getAbsolutePath());
265 propFile.deleteOnExit();
266 } catch (final IOException e) {
267 logger.warn("Failed to create system properties file.", e);
268 }
269 }
270
271 try {
272 crawlingInfoHelper.store(options.sessionId, true);
273 final String dayForCleanupStr;
274 int dayForCleanup = -1;
275 if (StringUtil.isNotBlank(options.expires)) {
276 dayForCleanupStr = options.expires;
277 try {
278 dayForCleanup = Integer.parseInt(dayForCleanupStr);
279 } catch (final NumberFormatException e) {}
280 } else {
281 dayForCleanup = ComponentUtil.getFessConfig().getDayForCleanup();
282 }
283 crawlingInfoHelper.updateParams(options.sessionId, options.name, dayForCleanup);
284 } catch (final Exception e) {
285 logger.warn("Failed to store crawling information.", e);
286 }
287
288 try {
289 return crawler.doCrawl(options);
290 } finally {
291 try {
292 crawlingInfoHelper.store(options.sessionId, false);
293 } catch (final Exception e) {
294 logger.warn("Failed to store crawling information.", e);
295 }
296
297 final Map<String, String> infoMap = crawlingInfoHelper.getInfoMap(options.sessionId);
298
299 final StringBuilder buf = new StringBuilder(500);
300 for (final Map.Entry<String, String> entry : infoMap.entrySet()) {
301 if (buf.length() != 0) {
302 buf.append(',');
303 }
304 buf.append(entry.getKey()).append('=').append(entry.getValue());
305 }
306 logger.info("[CRAWL INFO] " + buf.toString());
307
308
309 try {
310 crawler.sendMail(infoMap);
311 } catch (final Exception e) {
312 logger.warn("Failed to send a mail.", e);
313 }
314
315 }
316 }
317
318 protected void sendMail(final Map<String, String> infoMap) {
319 final FessConfig fessConfig = ComponentUtil.getFessConfig();
320 final String toStrs = fessConfig.getNotificationTo();
321 if (StringUtil.isNotBlank(toStrs)) {
322 final String[] toAddresses = toStrs.split(",");
323 final Map<String, String> dataMap = new HashMap<>();
324 for (final Map.Entry<String, String> entry : infoMap.entrySet()) {
325 dataMap.put(StringUtil.decapitalize(entry.getKey()), entry.getValue());
326 }
327
328 dataMap.put("hostname", ComponentUtil.getSystemHelper().getHostname());
329
330 logger.debug("\ninfoMap: {}\ndataMap: {}", infoMap, dataMap);
331
332 final Postbox postbox = ComponentUtil.getComponent(Postbox.class);
333 CrawlerPostcard.droppedInto(postbox, postcard -> {
334 postcard.setFrom(fessConfig.getMailFromAddress(), fessConfig.getMailFromName());
335 postcard.addReplyTo(fessConfig.getMailReturnPath());
336 stream(toAddresses).of(stream -> stream.forEach(address -> {
337 postcard.addTo(address);
338 }));
339 postcard.setCrawlerEndTime(getValueFromMap(dataMap, "crawlerEndTime", StringUtil.EMPTY));
340 postcard.setCrawlerExecTime(getValueFromMap(dataMap, "crawlerExecTime", "0"));
341 postcard.setCrawlerStartTime(getValueFromMap(dataMap, "crawlerStartTime", StringUtil.EMPTY));
342 postcard.setDataCrawlEndTime(getValueFromMap(dataMap, "dataCrawlEndTime", StringUtil.EMPTY));
343 postcard.setDataCrawlExecTime(getValueFromMap(dataMap, "dataCrawlExecTime", "0"));
344 postcard.setDataCrawlStartTime(getValueFromMap(dataMap, "dataCrawlStartTime", StringUtil.EMPTY));
345 postcard.setDataIndexSize(getValueFromMap(dataMap, "dataIndexSize", "0"));
346 postcard.setDataIndexExecTime(getValueFromMap(dataMap, "dataIndexExecTime", "0"));
347 postcard.setHostname(getValueFromMap(dataMap, "hostname", StringUtil.EMPTY));
348 postcard.setWebFsCrawlEndTime(getValueFromMap(dataMap, "webFsCrawlEndTime", StringUtil.EMPTY));
349 postcard.setWebFsCrawlExecTime(getValueFromMap(dataMap, "webFsCrawlExecTime", "0"));
350 postcard.setWebFsCrawlStartTime(getValueFromMap(dataMap, "webFsCrawlStartTime", StringUtil.EMPTY));
351 postcard.setWebFsIndexExecTime(getValueFromMap(dataMap, "webFsIndexExecTime", "0"));
352 postcard.setWebFsIndexSize(getValueFromMap(dataMap, "webFsIndexSize", "0"));
353 if (Constants.TRUE.equalsIgnoreCase(infoMap.get(Constants.CRAWLER_STATUS))) {
354 postcard.setStatus(Constants.OK);
355 } else {
356 postcard.setStatus(Constants.FAIL);
357 }
358 });
359 }
360 }
361
362 private String getValueFromMap(final Map<String, String> dataMap, final String key, final String defaultValue) {
363 final String value = dataMap.get(key);
364 if (StringUtil.isBlank(value)) {
365 return defaultValue;
366 }
367 return value;
368 }
369
370 public int doCrawl(final Options options) {
371 if (logger.isInfoEnabled()) {
372 logger.info("Starting Crawler..");
373 }
374
375 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
376
377 final long totalTime = System.currentTimeMillis();
378
379 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
380
381 try {
382 writeTimeToSessionInfo(crawlingInfoHelper, Constants.CRAWLER_START_TIME);
383
384
385 final List<String> ptList = new ArrayList<>();
386 ptList.add(Constants.PROCESS_TYPE_CRAWLING);
387 ptList.add(Constants.PROCESS_TYPE_BOTH);
388 pathMappingHelper.setPathMappingList(options.sessionId, pathMappingService.getPathMappingList(ptList));
389
390
391 try {
392 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
393 duplicateHostHelper.init();
394 } catch (final Exception e) {
395 logger.warn("Could not initialize duplicateHostHelper.", e);
396 }
397
398
399 crawlingInfoService.deleteSessionIdsBefore(options.sessionId, options.name, ComponentUtil.getSystemHelper()
400 .getCurrentTimeAsLong());
401
402 final List<String> webConfigIdList = options.getWebConfigIdList();
403 final List<String> fileConfigIdList = options.getFileConfigIdList();
404 final List<String> dataConfigIdList = options.getDataConfigIdList();
405 final boolean runAll = webConfigIdList == null && fileConfigIdList == null && dataConfigIdList == null;
406
407 Thread webFsCrawlerThread = null;
408 Thread dataCrawlerThread = null;
409
410 if (runAll || webConfigIdList != null || fileConfigIdList != null) {
411 webFsCrawlerThread = new Thread((Runnable) () -> {
412
413 writeTimeToSessionInfo(crawlingInfoHelper, Constants.WEB_FS_CRAWLER_START_TIME);
414 webFsIndexHelper.crawl(options.sessionId, webConfigIdList, fileConfigIdList);
415 writeTimeToSessionInfo(crawlingInfoHelper, Constants.WEB_FS_CRAWLER_END_TIME);
416 }, WEB_FS_CRAWLING_PROCESS);
417 webFsCrawlerThread.start();
418 }
419
420 if (runAll || dataConfigIdList != null) {
421 dataCrawlerThread = new Thread((Runnable) () -> {
422
423 writeTimeToSessionInfo(crawlingInfoHelper, Constants.DATA_CRAWLER_START_TIME);
424 dataIndexHelper.crawl(options.sessionId, dataConfigIdList);
425 writeTimeToSessionInfo(crawlingInfoHelper, Constants.DATA_CRAWLER_END_TIME);
426 }, DATA_CRAWLING_PROCESS);
427 dataCrawlerThread.start();
428 }
429
430 joinCrawlerThread(webFsCrawlerThread);
431 joinCrawlerThread(dataCrawlerThread);
432
433 if (logger.isInfoEnabled()) {
434 logger.info("Finished Crawler");
435 }
436
437 return Constants.EXIT_OK;
438 } catch (final Throwable t) {
439 logger.warn("An exception occurs on the crawl task.", t);
440 return Constants.EXIT_FAIL;
441 } finally {
442 pathMappingHelper.removePathMappingList(options.sessionId);
443 crawlingInfoHelper.putToInfoMap(Constants.CRAWLER_STATUS, errors.isEmpty() ? Constants.T.toString() : Constants.F.toString());
444 if (!errors.isEmpty()) {
445 crawlingInfoHelper.putToInfoMap(Constants.CRAWLER_ERRORS, errors.stream().map(s -> s.replace(" ", StringUtil.EMPTY))
446 .collect(Collectors.joining(" ")));
447 }
448 writeTimeToSessionInfo(crawlingInfoHelper, Constants.CRAWLER_END_TIME);
449 crawlingInfoHelper.putToInfoMap(Constants.CRAWLER_EXEC_TIME, Long.toString(System.currentTimeMillis() - totalTime));
450
451 }
452 }
453
454 protected void writeTimeToSessionInfo(final CrawlingInfoHelper crawlingInfoHelper, final String key) {
455 if (crawlingInfoHelper != null) {
456 final SimpleDateFormat dateFormat = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
457 crawlingInfoHelper.putToInfoMap(key, dateFormat.format(new Date()));
458 }
459 }
460
461 private void joinCrawlerThread(final Thread crawlerThread) {
462 if (crawlerThread != null) {
463 try {
464 crawlerThread.join();
465 } catch (final Exception e) {
466 logger.info("Interrupted a crawling process: " + crawlerThread.getName());
467 }
468 }
469 }
470 }