1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.job;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.File;
21 import java.text.SimpleDateFormat;
22 import java.util.ArrayList;
23 import java.util.Date;
24 import java.util.List;
25 import java.util.concurrent.atomic.AtomicInteger;
26
27 import javax.servlet.ServletContext;
28
29 import org.apache.commons.lang3.StringUtils;
30 import org.apache.commons.lang3.SystemUtils;
31 import org.apache.logging.log4j.LogManager;
32 import org.apache.logging.log4j.Logger;
33 import org.codelibs.core.lang.StringUtil;
34 import org.codelibs.core.timer.TimeoutTask;
35 import org.codelibs.fess.Constants;
36 import org.codelibs.fess.es.config.exbhv.ScheduledJobBhv;
37 import org.codelibs.fess.exception.JobProcessingException;
38 import org.codelibs.fess.exec.Crawler;
39 import org.codelibs.fess.helper.ProcessHelper;
40 import org.codelibs.fess.helper.SystemHelper;
41 import org.codelibs.fess.mylasta.direction.FessConfig;
42 import org.codelibs.fess.util.ComponentUtil;
43 import org.codelibs.fess.util.InputStreamThread;
44 import org.codelibs.fess.util.JobProcess;
45 import org.codelibs.fess.util.ResourceUtil;
46
47 public class CrawlJob extends ExecJob {
48
49 private static final Logger logger = LogManager.getLogger(CrawlJob.class);
50
51 protected String namespace = Constants.CRAWLING_INFO_SYSTEM_NAME;
52
53 protected String[] webConfigIds;
54
55 protected String[] fileConfigIds;
56
57 protected String[] dataConfigIds;
58
59 protected int documentExpires = -2;
60
61 public CrawlJob namespace(final String namespace) {
62 this.namespace = namespace;
63 return this;
64 }
65
66 public CrawlJob documentExpires(final int documentExpires) {
67 this.documentExpires = documentExpires;
68 return this;
69 }
70
71 public CrawlJob webConfigIds(final String[] webConfigIds) {
72 this.webConfigIds = webConfigIds;
73 return this;
74 }
75
76 public CrawlJob fileConfigIds(final String[] fileConfigIds) {
77 this.fileConfigIds = fileConfigIds;
78 return this;
79 }
80
81 public CrawlJob dataConfigIds(final String[] dataConfigIds) {
82 this.dataConfigIds = dataConfigIds;
83 return this;
84 }
85
86 @Override
87 public String execute() {
88
89 final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger();
90 if (maxCrawlerProcesses > 0) {
91 final int runningJobCount = getRunningJobCount();
92 if (runningJobCount > maxCrawlerProcesses) {
93 throw new JobProcessingException(
94 runningJobCount + " crawler processes are running. Max processes are " + maxCrawlerProcesses + ".");
95 }
96 }
97
98 final StringBuilder resultBuf = new StringBuilder(100);
99 final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
100
101 if (sessionId == null) {
102 final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
103 sessionId = sdf.format(new Date());
104 }
105 resultBuf.append("Session Id: ").append(sessionId).append("\n");
106 resultBuf.append("Web Config Id:");
107 if (webConfigIds == null) {
108 if (runAll) {
109 resultBuf.append(" ALL\n");
110 } else {
111 resultBuf.append(" NONE\n");
112 }
113 } else {
114 for (final String id : webConfigIds) {
115 resultBuf.append(' ').append(id);
116 }
117 resultBuf.append('\n');
118 }
119 resultBuf.append("File Config Id:");
120 if (fileConfigIds == null) {
121 if (runAll) {
122 resultBuf.append(" ALL\n");
123 } else {
124 resultBuf.append(" NONE\n");
125 }
126 } else {
127 for (final String id : fileConfigIds) {
128 resultBuf.append(' ').append(id);
129 }
130 resultBuf.append('\n');
131 }
132 resultBuf.append("Data Config Id:");
133 if (dataConfigIds == null) {
134 if (runAll) {
135 resultBuf.append(" ALL\n");
136 } else {
137 resultBuf.append(" NONE\n");
138 }
139 } else {
140 for (final String id : dataConfigIds) {
141 resultBuf.append(' ').append(id);
142 }
143 resultBuf.append('\n');
144 }
145
146 if (jobExecutor != null) {
147 jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
148 }
149
150 final TimeoutTask timeoutTask = createTimeoutTask();
151 try {
152 executeCrawler();
153 ComponentUtil.getKeyMatchHelper().update();
154 } catch (final JobProcessingException e) {
155 throw e;
156 } catch (final Exception e) {
157 throw new JobProcessingException("Failed to execute a crawl job.", e);
158 } finally {
159 if (timeoutTask != null && !timeoutTask.isCanceled()) {
160 timeoutTask.cancel();
161 }
162 }
163
164 return resultBuf.toString();
165
166 }
167
168 protected int getRunningJobCount() {
169 final AtomicInteger counter = new AtomicInteger(0);
170 final FessConfig fessConfig = ComponentUtil.getFessConfig();
171 ComponentUtil.getComponent(ScheduledJobBhv.class).selectCursor(cb -> {
172 cb.query().setAvailable_Equal(Constants.T);
173 cb.query().setCrawler_Equal(Constants.T);
174 }, scheduledJob -> {
175 if (fessConfig.isSchedulerTarget(scheduledJob.getTarget())) {
176 if (scheduledJob.isRunning()) {
177 if (logger.isDebugEnabled()) {
178 logger.debug("{} is running.", scheduledJob.getId());
179 }
180 counter.incrementAndGet();
181 } else if (logger.isDebugEnabled()) {
182 logger.debug("{} is not running.", scheduledJob.getId());
183 }
184 }
185 });
186 return counter.get();
187 }
188
189 protected void executeCrawler() {
190 final List<String> cmdList = new ArrayList<>();
191 final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
192 final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
193 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
194 final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
195 final FessConfig fessConfig = ComponentUtil.getFessConfig();
196
197 cmdList.add(fessConfig.getJavaCommandPath());
198
199
200 cmdList.add("-cp");
201 final StringBuilder buf = new StringBuilder(100);
202 ResourceUtil.getOverrideConfPath().ifPresent(p -> {
203 buf.append(p);
204 buf.append(cpSeparator);
205 });
206 final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
207 if (StringUtil.isNotBlank(confPath)) {
208 buf.append(confPath);
209 buf.append(cpSeparator);
210 }
211
212 buf.append("WEB-INF");
213 buf.append(File.separator);
214 buf.append("env");
215 buf.append(File.separator);
216 buf.append(getExecuteType());
217 buf.append(File.separator);
218 buf.append("resources");
219 buf.append(cpSeparator);
220
221 buf.append("WEB-INF");
222 buf.append(File.separator);
223 buf.append("classes");
224
225 final String userDir = System.getProperty("user.dir");
226 final File targetDir = new File(userDir, "target");
227 final File targetClassesDir = new File(targetDir, "classes");
228 if (targetClassesDir.isDirectory()) {
229 buf.append(cpSeparator);
230 buf.append(targetClassesDir.getAbsolutePath());
231 }
232
233 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")),
234 "WEB-INF" + File.separator + "lib" + File.separator);
235
236 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")),
237 "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
238
239 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")),
240 "WEB-INF" + File.separator + "plugin" + File.separator);
241 final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
242 if (targetLibDir.isDirectory()) {
243 appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
244 }
245 cmdList.add(buf.toString());
246
247 if (useLocalFesen) {
248 final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
249 if (StringUtil.isNotBlank(httpAddress)) {
250 cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
251 }
252 }
253
254 final String systemLastaEnv = System.getProperty("lasta.env");
255 if (StringUtil.isNotBlank(systemLastaEnv)) {
256 if ("web".equals(systemLastaEnv)) {
257 cmdList.add("-Dlasta.env=" + getExecuteType());
258 } else {
259 cmdList.add("-Dlasta.env=" + systemLastaEnv);
260 }
261 } else if (StringUtil.isNotBlank(lastaEnv)) {
262 cmdList.add("-Dlasta.env=" + lastaEnv);
263 } else {
264 cmdList.add("-Dlasta.env=" + getExecuteType());
265 }
266
267 addFessConfigProperties(cmdList);
268 addFessSystemProperties(cmdList);
269 addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
270 cmdList.add("-Dfess." + getExecuteType() + ".process=true");
271 cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
272 addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
273 if (logLevel == null) {
274 addSystemProperty(cmdList, "fess.log.level", null, null);
275 } else {
276 cmdList.add("-Dfess.log.level=" + logLevel);
277 if ("debug".equalsIgnoreCase(logLevel)) {
278 cmdList.add("-Dorg.apache.tika.service.error.warn=true");
279 }
280 }
281 stream(fessConfig.getJvmCrawlerOptionsAsArray())
282 .of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
283
284 File ownTmpDir = null;
285 final String tmpDir = System.getProperty("java.io.tmpdir");
286 if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
287 ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
288 if (ownTmpDir.mkdirs()) {
289 cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
290 cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
291 } else {
292 ownTmpDir = null;
293 }
294 }
295
296 cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
297
298 if (!jvmOptions.isEmpty()) {
299 jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
300 }
301
302 cmdList.add(Crawler.class.getCanonicalName());
303
304 cmdList.add("--sessionId");
305 cmdList.add(sessionId);
306 cmdList.add("--name");
307 cmdList.add(namespace);
308
309 if (webConfigIds != null && webConfigIds.length > 0) {
310 cmdList.add("-w");
311 cmdList.add(StringUtils.join(webConfigIds, ','));
312 }
313 if (fileConfigIds != null && fileConfigIds.length > 0) {
314 cmdList.add("-f");
315 cmdList.add(StringUtils.join(fileConfigIds, ','));
316 }
317 if (dataConfigIds != null && dataConfigIds.length > 0) {
318 cmdList.add("-d");
319 cmdList.add(StringUtils.join(dataConfigIds, ','));
320 }
321 if (documentExpires >= -1) {
322 cmdList.add("-e");
323 cmdList.add(Integer.toString(documentExpires));
324 }
325
326 final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
327 try {
328 cmdList.add("-p");
329 cmdList.add(propFile.getAbsolutePath());
330 createSystemProperties(cmdList, propFile);
331
332 final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
333
334 if (logger.isInfoEnabled()) {
335 logger.info("Crawler: \nDirectory={}\nOptions={}", baseDir, cmdList);
336 }
337
338 final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
339 pb.directory(baseDir);
340 pb.redirectErrorStream(true);
341 });
342
343 final InputStreamThread it = jobProcess.getInputStreamThread();
344 it.start();
345
346 final Process currentProcess = jobProcess.getProcess();
347 currentProcess.waitFor();
348 it.join(5000);
349
350 final int exitValue = currentProcess.exitValue();
351
352 if (logger.isInfoEnabled()) {
353 logger.info("Crawler: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
354 }
355 if (exitValue != 0) {
356 final StringBuilder out = new StringBuilder();
357 if (processTimeout) {
358 out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
359 }
360 out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
361 throw new JobProcessingException(out.toString());
362 }
363 } catch (final JobProcessingException e) {
364 throw e;
365 } catch (final Exception e) {
366 throw new JobProcessingException("Crawler Process terminated.", e);
367 } finally {
368 try {
369 processHelper.destroyProcess(sessionId);
370 } finally {
371 if (propFile != null && !propFile.delete()) {
372 logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
373 }
374 deleteTempDir(ownTmpDir);
375 }
376 }
377 }
378
379 @Override
380 protected String getExecuteType() {
381 return Constants.EXECUTE_TYPE_CRAWLER;
382 }
383
384 }