1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.job;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20
21 import java.io.File;
22 import java.io.FileOutputStream;
23 import java.io.FilenameFilter;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Date;
27 import java.util.List;
28 import java.util.Properties;
29
30 import javax.servlet.ServletContext;
31
32 import org.apache.commons.io.FileUtils;
33 import org.apache.commons.lang3.SystemUtils;
34 import org.codelibs.core.lang.StringUtil;
35 import org.codelibs.fess.Constants;
36 import org.codelibs.fess.exception.FessSystemException;
37 import org.codelibs.fess.exec.SuggestCreator;
38 import org.codelibs.fess.helper.ProcessHelper;
39 import org.codelibs.fess.mylasta.direction.FessConfig;
40 import org.codelibs.fess.util.ComponentUtil;
41 import org.codelibs.fess.util.InputStreamThread;
42 import org.codelibs.fess.util.JobProcess;
43 import org.slf4j.Logger;
44 import org.slf4j.LoggerFactory;
45
46 public class SuggestJob {
47 private static final String REMOTE_DEBUG_OPTIONS = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=localhost:8000";
48
49 private static final Logger logger = LoggerFactory.getLogger(SuggestJob.class);
50
51 protected JobExecutor jobExecutor;
52
53 protected String sessionId;
54
55 protected boolean useLocaleElasticsearch = true;
56
57 protected String logFilePath;
58
59 protected String logLevel;
60
61 protected String jvmOptions;
62
63 public SuggestJob jobExecutor(final JobExecutor jobExecutor) {
64 this.jobExecutor = jobExecutor;
65 return this;
66 }
67
68 public SuggestJob sessionId(final String sessionId) {
69 this.sessionId = sessionId;
70 return this;
71 }
72
73 public SuggestJob logFilePath(final String logFilePath) {
74 this.logFilePath = logFilePath;
75 return this;
76 }
77
78 public SuggestJob logLevel(final String logLevel) {
79 this.logLevel = logLevel;
80 return this;
81 }
82
83 public SuggestJob useLocaleElasticsearch(final boolean useLocaleElasticsearch) {
84 this.useLocaleElasticsearch = useLocaleElasticsearch;
85 return this;
86 }
87
88 public SuggestJob remoteDebug() {
89 return jvmOptions(REMOTE_DEBUG_OPTIONS);
90 }
91
92 public SuggestJob jvmOptions(final String option) {
93 this.jvmOptions = option;
94 return this;
95 }
96
97 public String execute(final JobExecutor jobExecutor) {
98 jobExecutor(jobExecutor);
99 return execute();
100 }
101
102 public String execute() {
103 final StringBuilder resultBuf = new StringBuilder();
104
105 if (sessionId == null) {
106 final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
107 sessionId = sdf.format(new Date());
108 }
109 resultBuf.append("Session Id: ").append(sessionId).append("\n");
110 if (jobExecutor != null) {
111 jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
112 }
113
114 try {
115 executeSuggestCreator();
116 } catch (final Exception e) {
117 logger.error("Failed to purge user info.", e);
118 resultBuf.append(e.getMessage()).append("\n");
119 }
120
121 return resultBuf.toString();
122
123 }
124
125 protected void executeSuggestCreator() {
126 final List<String> cmdList = new ArrayList<>();
127 final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
128 final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
129 final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
130 final FessConfig fessConfig = ComponentUtil.getFessConfig();
131
132 cmdList.add(fessConfig.getJavaCommandPath());
133
134
135 cmdList.add("-cp");
136 final StringBuilder buf = new StringBuilder(100);
137 final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
138 if (StringUtil.isNotBlank(confPath)) {
139 buf.append(confPath);
140 buf.append(cpSeparator);
141 }
142
143 buf.append("WEB-INF");
144 buf.append(File.separator);
145 buf.append("suggest");
146 buf.append(File.separator);
147 buf.append("resources");
148 buf.append(cpSeparator);
149
150 buf.append("WEB-INF");
151 buf.append(File.separator);
152 buf.append("classes");
153
154 final String userDir = System.getProperty("user.dir");
155 final File targetDir = new File(userDir, "target");
156 final File targetClassesDir = new File(targetDir, "classes");
157 if (targetClassesDir.isDirectory()) {
158 buf.append(cpSeparator);
159 buf.append(targetClassesDir.getAbsolutePath());
160 }
161
162 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
163
164 appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/suggest/lib")), "WEB-INF/suggest" + File.separator
165 + "lib" + File.separator);
166 final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
167 if (targetLibDir.isDirectory()) {
168 appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
169 }
170 cmdList.add(buf.toString());
171
172 if (useLocaleElasticsearch) {
173 final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
174 if (StringUtil.isNotBlank(transportAddresses)) {
175 cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
176 }
177 }
178
179 final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
180 if (StringUtil.isNotBlank(clusterName)) {
181 cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
182 } else {
183 cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + fessConfig.getElasticsearchClusterName());
184 }
185
186 final String lastaEnv = System.getProperty("lasta.env");
187 if (StringUtil.isNotBlank(lastaEnv)) {
188 if (lastaEnv.equals("web")) {
189 cmdList.add("-Dlasta.env=suggest");
190 } else {
191 cmdList.add("-Dlasta.env=" + lastaEnv);
192 }
193 }
194
195 cmdList.add("-Dfess.suggest.process=true");
196 if (logFilePath == null) {
197 final String value = System.getProperty("fess.log.path");
198 logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
199 }
200 cmdList.add("-Dfess.log.path=" + logFilePath);
201 addSystemProperty(cmdList, "fess.log.name", "fess-suggest", "-suggest");
202 if (logLevel == null) {
203 addSystemProperty(cmdList, "fess.log.level", null, null);
204 } else {
205 cmdList.add("-Dfess.log.level=" + logLevel);
206 }
207 stream(fessConfig.getJvmSuggestOptionsAsArray()).of(
208 stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
209
210 File ownTmpDir = null;
211 final String tmpDir = System.getProperty("java.io.tmpdir");
212 if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
213 ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
214 if (ownTmpDir.mkdirs()) {
215 cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
216 } else {
217 ownTmpDir = null;
218 }
219 }
220
221 if (StringUtil.isNotBlank(jvmOptions)) {
222 split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
223 }
224
225 cmdList.add(SuggestCreator.class.getCanonicalName());
226
227 cmdList.add("--sessionId");
228 cmdList.add(sessionId);
229
230 File propFile = null;
231 try {
232 cmdList.add("-p");
233 propFile = File.createTempFile("crawler_", ".properties");
234 cmdList.add(propFile.getAbsolutePath());
235 try (FileOutputStream out = new FileOutputStream(propFile)) {
236 final Properties prop = new Properties();
237 prop.putAll(ComponentUtil.getSystemProperties());
238 prop.store(out, cmdList.toString());
239 }
240
241 final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
242
243 if (logger.isInfoEnabled()) {
244 logger.info("SuggestCreator: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
245 }
246
247 final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
248 pb.directory(baseDir);
249 pb.redirectErrorStream(true);
250 });
251
252 final InputStreamThread it = jobProcess.getInputStreamThread();
253 it.start();
254
255 final Process currentProcess = jobProcess.getProcess();
256 currentProcess.waitFor();
257 it.join(5000);
258
259 final int exitValue = currentProcess.exitValue();
260
261 if (logger.isInfoEnabled()) {
262 logger.info("SuggestCreator: Exit Code=" + exitValue + " - SuggestCreator Process Output:\n" + it.getOutput());
263 }
264 if (exitValue != 0) {
265 throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
266 }
267 ComponentUtil.getPopularWordHelper().clearCache();
268 } catch (final FessSystemException e) {
269 throw e;
270 } catch (final InterruptedException e) {
271 logger.warn("SuggestCreator Process interrupted.");
272 } catch (final Exception e) {
273 throw new FessSystemException("SuggestCreator Process terminated.", e);
274 } finally {
275 try {
276 processHelper.destroyProcess(sessionId);
277 } finally {
278 if (propFile != null && !propFile.delete()) {
279 logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
280 }
281 deleteTempDir(ownTmpDir);
282 }
283 }
284 }
285
286 private void addSystemProperty(final List<String> crawlerCmdList, final String name, final String defaultValue, final String appendValue) {
287 final String value = System.getProperty(name);
288 if (value != null) {
289 final StringBuilder buf = new StringBuilder();
290 buf.append("-D").append(name).append("=").append(value);
291 if (appendValue != null) {
292 buf.append(appendValue);
293 }
294 crawlerCmdList.add(buf.toString());
295 } else if (defaultValue != null) {
296 crawlerCmdList.add("-D" + name + "=" + defaultValue);
297 }
298 }
299
300 protected void deleteTempDir(final File ownTmpDir) {
301 if (ownTmpDir == null) {
302 return;
303 }
304 if (!FileUtils.deleteQuietly(ownTmpDir)) {
305 logger.warn("Could not delete a temp dir: " + ownTmpDir.getAbsolutePath());
306 }
307 }
308
309 protected void appendJarFile(final String cpSeparator, final StringBuilder buf, final File libDir, final String basePath) {
310 final File[] jarFiles = libDir.listFiles((FilenameFilter) (dir, name) -> name.toLowerCase().endsWith(".jar"));
311 if (jarFiles != null) {
312 for (final File file : jarFiles) {
313 buf.append(cpSeparator);
314 buf.append(basePath);
315 buf.append(file.getName());
316 }
317 }
318 }
319 }