1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.job;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.FilenameFilter;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28 import java.util.Properties;
29
30 import javax.servlet.ServletContext;
31
32 import org.apache.commons.io.FileUtils;
33 import org.apache.commons.lang3.StringUtils;
34 import org.apache.commons.lang3.SystemUtils;
35 import org.codelibs.core.lang.StringUtil;
36 import org.codelibs.fess.Constants;
37 import org.codelibs.fess.exception.FessSystemException;
38 import org.codelibs.fess.exec.Crawler;
39 import org.codelibs.fess.helper.ProcessHelper;
40 import org.codelibs.fess.helper.SystemHelper;
41 import org.codelibs.fess.mylasta.direction.FessConfig;
42 import org.codelibs.fess.util.ComponentUtil;
43 import org.codelibs.fess.util.InputStreamThread;
44 import org.codelibs.fess.util.JobProcess;
45 import org.slf4j.Logger;
46 import org.slf4j.LoggerFactory;
47
48 public class CrawlJob {
49 private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
50
51 private static final Logger logger = LoggerFactory.getLogger(CrawlJob.class);
52
53 protected JobExecutor jobExecutor;
54
55 protected String sessionId;
56
57 protected String namespace = Constants.CRAWLING_INFO_SYSTEM_NAME;
58
59 protected String[] webConfigIds;
60
61 protected String[] fileConfigIds;
62
63 protected String[] dataConfigIds;
64
65 protected String logFilePath;
66
67 protected String logLevel;
68
69 protected int documentExpires = -2;
70
71 protected boolean useLocalElasticsearch = true;
72
73 protected String jvmOptions;
74
75 protected String lastaEnv;
76
77 public CrawlJob jobExecutor(final JobExecutor jobExecutor) {
78 this.jobExecutor = jobExecutor;
79 return this;
80 }
81
82 public CrawlJob sessionId(final String sessionId) {
83 this.sessionId = sessionId;
84 return this;
85 }
86
87 public CrawlJob namespace(final String namespace) {
88 this.namespace = namespace;
89 return this;
90 }
91
92 public CrawlJob logFilePath(final String logFilePath) {
93 this.logFilePath = logFilePath;
94 return this;
95 }
96
97 public CrawlJob logLevel(final String logLevel) {
98 this.logLevel = logLevel;
99 return this;
100 }
101
102 public CrawlJob documentExpires(final int documentExpires) {
103 this.documentExpires = documentExpires;
104 return this;
105 }
106
107 public CrawlJob webConfigIds(final String[] webConfigIds) {
108 this.webConfigIds = webConfigIds;
109 return this;
110 }
111
112 public CrawlJob fileConfigIds(final String[] fileConfigIds) {
113 this.fileConfigIds = fileConfigIds;
114 return this;
115 }
116
117 public CrawlJob dataConfigIds(final String[] dataConfigIds) {
118 this.dataConfigIds = dataConfigIds;
119 return this;
120 }
121
122 public CrawlJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
123 this.useLocalElasticsearch = useLocaleElasticsearch;
124 return this;
125 }
126
127 public CrawlJob remoteDebug() {
128 return jvmOptions(REMOTE_DEBUG_OPTIONS);
129 }
130
131 public CrawlJob jvmOptions(final String option) {
132 this.jvmOptions = option;
133 return this;
134 }
135
136 public CrawlJob lastaEnv(final String env) {
137 this.lastaEnv = env;
138 return this;
139 }
140
141 public String execute(final JobExecutor jobExecutor) {
142 jobExecutor(jobExecutor);
143 return execute();
144 }
145
146 public String execute() {
147 final StringBuilder resultBuf = new StringBuilder(100);
148 final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
149
150 if (sessionId == null) {
151 final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
152 sessionId = sdf.format(new Date());
153 }
154 resultBuf.append("Session Id: ").append(sessionId).append("\n");
155 resultBuf.append("Web Config Id:");
156 if (webConfigIds == null) {
157 if (runAll) {
158 resultBuf.append(" ALL\n");
159 } else {
160 resultBuf.append(" NONE\n");
161 }
162 } else {
163 for (final String id : webConfigIds) {
164 resultBuf.append(' ').append(id);
165 }
166 resultBuf.append('\n');
167 }
168 resultBuf.append("File Config Id:");
169 if (fileConfigIds == null) {
170 if (runAll) {
171 resultBuf.append(" ALL\n");
172 } else {
173 resultBuf.append(" NONE\n");
174 }
175 } else {
176 for (final String id : fileConfigIds) {
177 resultBuf.append(' ').append(id);
178 }
179 resultBuf.append('\n');
180 }
181 resultBuf.append("Data Config Id:");
182 if (dataConfigIds == null) {
183 if (runAll) {
184 resultBuf.append(" ALL\n");
185 } else {
186 resultBuf.append(" NONE\n");
187 }
188 } else {
189 for (final String id : dataConfigIds) {
190 resultBuf.append(' ').append(id);
191 }
192 resultBuf.append('\n');
193 }
194
195 if (jobExecutor != null) {
196 jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
197 }
198
199 try {
200 executeCrawler();
201 ComponentUtil.getKeyMatchHelper().update();
202 } catch (final FessSystemException e) {
203 throw e;
204 } catch (final Exception e) {
205 throw new FessSystemException("Failed to execute a crawl job.", e);
206 }
207
208 return resultBuf.toString();
209
210 }
211
212 protected void executeCrawler() {
213 final List<String> cmdList = new ArrayList<>();
214 final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
215 final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
216 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
217 final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
218 final FessConfig fessConfig = ComponentUtil.getFessConfig();
219
220 cmdList.add(fessConfig.getJavaCommandPath());
221
222
223 cmdList.add("-cp");
224 final StringBuilder buf = new StringBuilder(100);
225 final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
226 if (StringUtil.isNotBlank(confPath)) {
227 buf.append(confPath);
228 buf.append(cpSeparator);
229 }
230
231 buf.append("WEB-INF");
232 buf.append(File.separator);
233 buf.append("crawler");
234 buf.append(File.separator);
235 buf.append("resources");
236 buf.append(cpSeparator);
237
238 buf.append("WEB-INF");
239 buf.append(File.separator);
240 buf.append("classes");
241
242 final String userDir = System.getProperty("user.dir");
243 final File targetDir = new File(userDir, "target");
244 final File targetClassesDir = new File(targetDir, "classes");
245 if (targetClassesDir.isDirectory()) {
246 buf.append(cpSeparator);
247 buf.append(targetClassesDir.getAbsolutePath());
248 }
249
250 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
251
252 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/crawler/lib")), "WEB-INF/crawler" + File.separator
253 + "lib" + File.separator);
254 final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
255 if (targetLibDir.isDirectory()) {
256 appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
257 }
258 cmdList.add(buf.toString());
259
260 if (useLocalElasticsearch) {
261 final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
262 if (StringUtil.isNotBlank(transportAddresses)) {
263 cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
264 }
265 }
266
267 final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
268 if (StringUtil.isNotBlank(clusterName)) {
269 cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
270 } else {
271 cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
272 }
273
274 final String systemLastaEnv = System.getProperty("lasta.env");
275 if (StringUtil.isNotBlank(systemLastaEnv)) {
276 if (systemLastaEnv.equals("web")) {
277 cmdList.add("-Dlasta.env=crawler");
278 } else {
279 cmdList.add("-Dlasta.env=" + systemLastaEnv);
280 }
281 } else if (StringUtil.isNotBlank(lastaEnv)) {
282 cmdList.add("-Dlasta.env=" + lastaEnv);
283 }
284
285 cmdList.add("-Dfess.crawler.process=true");
286 cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
287 addSystemProperty(cmdList, "fess.log.name", "fess-crawler", "-crawler");
288 if (logLevel == null) {
289 addSystemProperty(cmdList, "fess.log.level", null, null);
290 } else {
291 cmdList.add("-Dfess.log.level=" + logLevel);
292 }
293 stream(fessConfig.getJvmCrawlerOptionsAsArray()).of(
294 stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
295
296 File ownTmpDir = null;
297 final String tmpDir = System.getProperty("java.io.tmpdir");
298 if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
299 ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
300 if (ownTmpDir.mkdirs()) {
301 cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
302 cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
303 } else {
304 ownTmpDir = null;
305 }
306 }
307
308 cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
309
310 if (StringUtil.isNotBlank(jvmOptions)) {
311 split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
312 }
313
314 cmdList.add(Crawler.class.getCanonicalName());
315
316 cmdList.add("--sessionId");
317 cmdList.add(sessionId);
318 cmdList.add("--name");
319 cmdList.add(namespace);
320
321 if (webConfigIds != null && webConfigIds.length > 0) {
322 cmdList.add("-w");
323 cmdList.add(StringUtils.join(webConfigIds, ','));
324 }
325 if (fileConfigIds != null && fileConfigIds.length > 0) {
326 cmdList.add("-f");
327 cmdList.add(StringUtils.join(fileConfigIds, ','));
328 }
329 if (dataConfigIds != null && dataConfigIds.length > 0) {
330 cmdList.add("-d");
331 cmdList.add(StringUtils.join(dataConfigIds, ','));
332 }
333 if (documentExpires >= -1) {
334 cmdList.add("-e");
335 cmdList.add(Integer.toString(documentExpires));
336 }
337
338 File propFile = null;
339 try {
340 cmdList.add("-p");
341 propFile = File.createTempFile("crawler_", ".properties");
342 cmdList.add(propFile.getAbsolutePath());
343 try (FileOutputStream out = new FileOutputStream(propFile)) {
344 final Properties prop = new Properties();
345 prop.putAll(ComponentUtil.getSystemProperties());
346 prop.store(out, cmdList.toString());
347 }
348
349 final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
350
351 if (logger.isInfoEnabled()) {
352 logger.info("Crawler: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
353 }
354
355 final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
356 pb.directory(baseDir);
357 pb.redirectErrorStream(true);
358 });
359
360 final InputStreamThread it = jobProcess.getInputStreamThread();
361 it.start();
362
363 final Process currentProcess = jobProcess.getProcess();
364 currentProcess.waitFor();
365 it.join(5000);
366
367 final int exitValue = currentProcess.exitValue();
368
369 if (logger.isInfoEnabled()) {
370 logger.info("Crawler: Exit Code=" + exitValue + " - Crawler Process Output:\n" + it.getOutput());
371 }
372 if (exitValue != 0) {
373 throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
374 }
375 } catch (final FessSystemException e) {
376 throw e;
377 } catch (final InterruptedException e) {
378 logger.warn("Crawler Process interrupted.");
379 } catch (final Exception e) {
380 throw new FessSystemException("Crawler Process terminated.", e);
381 } finally {
382 try {
383 processHelper.destroyProcess(sessionId);
384 } finally {
385 if (propFile != null && !propFile.delete()) {
386 logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
387 }
388 deleteTempDir(ownTmpDir);
389 }
390 }
391 }
392
393 private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
394 final String value = System.getProperty(name);
395 if (value != null) {
396 final StringBuilder buf = new StringBuilder();
397 buf.append("-D").append(name).append("=").append(value);
398 if (appendValue != null) {
399 buf.append(appendValue);
400 }
401 crawlerCmdList.add(buf.toString());
402 } else if (defaultValue != null) {
403 crawlerCmdList.add("-D" + name + "=" + defaultValue);
404 }
405 }
406
407 protected void deleteTempDir(final File ownTmpDir) {
408 if (ownTmpDir == null) {
409 return;
410 }
411 if (!FileUtils.deleteQuietly(ownTmpDir)) {
412 logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
413 }
414 }
415
416 protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
417 final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
418 if (jarFiles != null) {
419 for (final File file : jarFiles) {
420 buf.append(cpSeparator);
421 buf.append(basePath);
422 buf.append(file.getName());
423 }
424 }
425 }
426 }