1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.job;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.File;
21 import java.text.SimpleDateFormat;
22 import java.util.ArrayList;
23 import java.util.Date;
24 import java.util.List;
25 import java.util.concurrent.atomic.AtomicInteger;
26
27 import org.apache.commons.lang3.StringUtils;
28 import org.apache.commons.lang3.SystemUtils;
29 import org.apache.logging.log4j.LogManager;
30 import org.apache.logging.log4j.Logger;
31 import org.codelibs.core.lang.StringUtil;
32 import org.codelibs.core.timer.TimeoutTask;
33 import org.codelibs.fess.Constants;
34 import org.codelibs.fess.exception.JobProcessingException;
35 import org.codelibs.fess.exec.Crawler;
36 import org.codelibs.fess.helper.ProcessHelper;
37 import org.codelibs.fess.helper.SystemHelper;
38 import org.codelibs.fess.mylasta.direction.FessConfig;
39 import org.codelibs.fess.opensearch.config.exbhv.ScheduledJobBhv;
40 import org.codelibs.fess.util.ComponentUtil;
41 import org.codelibs.fess.util.InputStreamThread;
42 import org.codelibs.fess.util.JobProcess;
43 import org.codelibs.fess.util.ResourceUtil;
44 import org.codelibs.fess.util.SystemUtil;
45
46 import jakarta.servlet.ServletContext;
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66 public class CrawlJob extends ExecJob {
67
68 private static final Logger logger = LogManager.getLogger(CrawlJob.class);
69
70
71
72
73
74
75 protected String namespace = Constants.CRAWLING_INFO_SYSTEM_NAME;
76
77
78
79
80
81 protected String[] webConfigIds;
82
83
84
85
86
87 protected String[] fileConfigIds;
88
89
90
91
92
93 protected String[] dataConfigIds;
94
95
96
97
98
99 protected int documentExpires = -2;
100
101
102
103
104
105
106 protected int hotThreadInterval = -1;
107
108
109
110
111
112 public CrawlJob() {
113 super();
114 }
115
116
117
118
119
120
121
122
123 public CrawlJob namespace(final String namespace) {
124 this.namespace = namespace;
125 return this;
126 }
127
128
129
130
131
132
133
134
135
136 public CrawlJob documentExpires(final int documentExpires) {
137 this.documentExpires = documentExpires;
138 return this;
139 }
140
141
142
143
144
145
146
147
148 public CrawlJob webConfigIds(final String[] webConfigIds) {
149 this.webConfigIds = webConfigIds;
150 return this;
151 }
152
153
154
155
156
157
158
159
160 public CrawlJob fileConfigIds(final String[] fileConfigIds) {
161 this.fileConfigIds = fileConfigIds;
162 return this;
163 }
164
165
166
167
168
169
170
171
172 public CrawlJob dataConfigIds(final String[] dataConfigIds) {
173 this.dataConfigIds = dataConfigIds;
174 return this;
175 }
176
177
178
179
180
181
182
183
184 public CrawlJob hotThread(final int hotThreadInterval) {
185 this.hotThreadInterval = hotThreadInterval;
186 return this;
187 }
188
189 @Override
190 public String execute() {
191
192 final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger();
193 if (maxCrawlerProcesses > 0) {
194 final int runningJobCount = getRunningJobCount();
195 if (runningJobCount > maxCrawlerProcesses) {
196 throw new JobProcessingException(
197 runningJobCount + " crawler processes are running. Max processes are " + maxCrawlerProcesses + ".");
198 }
199 }
200
201 final StringBuilder resultBuf = new StringBuilder(100);
202 final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
203
204 if (sessionId == null) {
205 final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
206 sessionId = sdf.format(new Date());
207 }
208 resultBuf.append("Session Id: ").append(sessionId).append("\n");
209 resultBuf.append("Web Config Id:");
210 if (webConfigIds == null) {
211 if (runAll) {
212 resultBuf.append(" ALL\n");
213 } else {
214 resultBuf.append(" NONE\n");
215 }
216 } else {
217 for (final String id : webConfigIds) {
218 resultBuf.append(' ').append(id);
219 }
220 resultBuf.append('\n');
221 }
222 resultBuf.append("File Config Id:");
223 if (fileConfigIds == null) {
224 if (runAll) {
225 resultBuf.append(" ALL\n");
226 } else {
227 resultBuf.append(" NONE\n");
228 }
229 } else {
230 for (final String id : fileConfigIds) {
231 resultBuf.append(' ').append(id);
232 }
233 resultBuf.append('\n');
234 }
235 resultBuf.append("Data Config Id:");
236 if (dataConfigIds == null) {
237 if (runAll) {
238 resultBuf.append(" ALL\n");
239 } else {
240 resultBuf.append(" NONE\n");
241 }
242 } else {
243 for (final String id : dataConfigIds) {
244 resultBuf.append(' ').append(id);
245 }
246 resultBuf.append('\n');
247 }
248
249 if (jobExecutor != null) {
250 jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
251 }
252
253 final TimeoutTask timeoutTask = createTimeoutTask();
254 try {
255 executeCrawler();
256 ComponentUtil.getKeyMatchHelper().update();
257 } catch (final JobProcessingException e) {
258 throw e;
259 } catch (final Exception e) {
260 throw new JobProcessingException("Failed to execute a crawl job.", e);
261 } finally {
262 if (timeoutTask != null && !timeoutTask.isCanceled()) {
263 timeoutTask.cancel();
264 }
265 }
266
267 return resultBuf.toString();
268
269 }
270
271
272
273
274
275
276
277
278 protected int getRunningJobCount() {
279 final AtomicInteger counter = new AtomicInteger(0);
280 final FessConfig fessConfig = ComponentUtil.getFessConfig();
281 ComponentUtil.getComponent(ScheduledJobBhv.class).selectCursor(cb -> {
282 cb.query().setAvailable_Equal(Constants.T);
283 cb.query().setCrawler_Equal(Constants.T);
284 }, scheduledJob -> {
285 if (fessConfig.isSchedulerTarget(scheduledJob.getTarget())) {
286 if (scheduledJob.isRunning()) {
287 if (logger.isDebugEnabled()) {
288 logger.debug("Scheduled job is running: id={}", scheduledJob.getId());
289 }
290 counter.incrementAndGet();
291 } else if (logger.isDebugEnabled()) {
292 logger.debug("Scheduled job is not running: id={}", scheduledJob.getId());
293 }
294 }
295 });
296 return counter.get();
297 }
298
299
300
301
302
303
304
305
306
307 protected void executeCrawler() {
308 final List<String> cmdList = new ArrayList<>();
309 final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
310 final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
311 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
312 final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
313 final FessConfig fessConfig = ComponentUtil.getFessConfig();
314
315 cmdList.add(fessConfig.getJavaCommandPath());
316
317
318 cmdList.add("-cp");
319 final StringBuilder buf = new StringBuilder(100);
320 ResourceUtil.getOverrideConfPath().ifPresent(p -> {
321 buf.append(p);
322 buf.append(cpSeparator);
323 });
324 final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
325 if (StringUtil.isNotBlank(confPath)) {
326 buf.append(confPath);
327 buf.append(cpSeparator);
328 }
329
330 buf.append("WEB-INF");
331 buf.append(File.separator);
332 buf.append("env");
333 buf.append(File.separator);
334 buf.append(getExecuteType());
335 buf.append(File.separator);
336 buf.append("resources");
337 buf.append(cpSeparator);
338
339 buf.append("WEB-INF");
340 buf.append(File.separator);
341 buf.append("classes");
342
343 final String userDir = System.getProperty("user.dir");
344 final File targetDir = new File(userDir, "target");
345 final File targetClassesDir = new File(targetDir, "classes");
346 if (targetClassesDir.isDirectory()) {
347 buf.append(cpSeparator);
348 buf.append(targetClassesDir.getAbsolutePath());
349 }
350
351 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")),
352 "WEB-INF" + File.separator + "lib" + File.separator);
353
354 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")),
355 "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
356
357 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")),
358 "WEB-INF" + File.separator + "plugin" + File.separator);
359 final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
360 if (targetLibDir.isDirectory()) {
361 appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
362 }
363 cmdList.add(buf.toString());
364
365 if (useLocalFesen) {
366 final String httpAddress = SystemUtil.getSearchEngineHttpAddress();
367 if (StringUtil.isNotBlank(httpAddress)) {
368 cmdList.add("-D" + Constants.FESS_SEARCH_ENGINE_HTTP_ADDRESS + "=" + httpAddress);
369 }
370 }
371
372 final String systemLastaEnv = System.getProperty("lasta.env");
373 if (StringUtil.isNotBlank(systemLastaEnv)) {
374 if ("web".equals(systemLastaEnv)) {
375 cmdList.add("-Dlasta.env=" + getExecuteType());
376 } else {
377 cmdList.add("-Dlasta.env=" + systemLastaEnv);
378 }
379 } else if (StringUtil.isNotBlank(lastaEnv)) {
380 cmdList.add("-Dlasta.env=" + lastaEnv);
381 } else {
382 cmdList.add("-Dlasta.env=" + getExecuteType());
383 }
384
385 addFessConfigProperties(cmdList);
386 addFessSystemProperties(cmdList);
387 addFessCustomSystemProperties(cmdList, fessConfig.getJobSystemPropertyFilterPattern());
388 addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
389 cmdList.add("-Dfess." + getExecuteType() + ".process=true");
390 cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
391 addSystemProperty(cmdList, "fess.log.name", getLogName("fess"), getLogName(StringUtil.EMPTY));
392 if (logLevel == null) {
393 addSystemProperty(cmdList, "fess.log.level", null, null);
394 } else {
395 cmdList.add("-Dfess.log.level=" + logLevel);
396 if ("debug".equalsIgnoreCase(logLevel)) {
397 cmdList.add("-Dorg.apache.tika.service.error.warn=true");
398 }
399 }
400 stream(fessConfig.getJvmCrawlerOptionsAsArray())
401 .of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
402
403 File ownTmpDir = null;
404 final String tmpDir = System.getProperty("java.io.tmpdir");
405 if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
406 ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
407 if (ownTmpDir.mkdirs()) {
408 cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
409 cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
410 } else {
411 ownTmpDir = null;
412 }
413 }
414
415 cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
416
417 if (!jvmOptions.isEmpty()) {
418 jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
419 }
420
421 cmdList.add(Crawler.class.getCanonicalName());
422
423 cmdList.add("--sessionId");
424 cmdList.add(sessionId);
425 cmdList.add("--name");
426 cmdList.add(namespace);
427
428 if (webConfigIds != null && webConfigIds.length > 0) {
429 cmdList.add("-w");
430 cmdList.add(StringUtils.join(webConfigIds, ','));
431 }
432 if (fileConfigIds != null && fileConfigIds.length > 0) {
433 cmdList.add("-f");
434 cmdList.add(StringUtils.join(fileConfigIds, ','));
435 }
436 if (dataConfigIds != null && dataConfigIds.length > 0) {
437 cmdList.add("-d");
438 cmdList.add(StringUtils.join(dataConfigIds, ','));
439 }
440 if (documentExpires >= -1) {
441 cmdList.add("-e");
442 cmdList.add(Integer.toString(documentExpires));
443 }
444 if (hotThreadInterval > -1) {
445 cmdList.add("-h");
446 cmdList.add(Integer.toString(hotThreadInterval));
447 }
448
449 final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
450 try {
451 cmdList.add("-p");
452 cmdList.add(propFile.getAbsolutePath());
453 createSystemProperties(cmdList, propFile);
454
455 final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
456
457 if (logger.isInfoEnabled()) {
458 logger.info("Crawler: \nDirectory={}\nOptions={}", baseDir, cmdList);
459 }
460
461 final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
462 pb.directory(baseDir);
463 pb.redirectErrorStream(true);
464 });
465
466 final InputStreamThread it = jobProcess.getInputStreamThread();
467 it.start();
468
469 final Process currentProcess = jobProcess.getProcess();
470 currentProcess.waitFor();
471 it.join(5000);
472
473 final int exitValue = currentProcess.exitValue();
474
475 if (logger.isInfoEnabled()) {
476 logger.info("Crawler: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
477 }
478 if (exitValue != 0) {
479 final StringBuilder out = new StringBuilder();
480 if (processTimeout) {
481 out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
482 }
483 out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
484 throw new JobProcessingException(out.toString());
485 }
486 } catch (final JobProcessingException e) {
487 throw e;
488 } catch (final Exception e) {
489 throw new JobProcessingException("Crawler Process terminated.", e);
490 } finally {
491 try {
492 processHelper.destroyProcess(sessionId);
493 } finally {
494 if (propFile != null && !propFile.delete()) {
495 logger.warn("Failed to delete properties file: {}", propFile.getAbsolutePath());
496 }
497 deleteTempDir(ownTmpDir);
498 }
499 }
500 }
501
502 @Override
503 protected String getExecuteType() {
504 return Constants.EXECUTE_TYPE_CRAWLER;
505 }
506
507 }