1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.exec;
17
18 import java.io.File;
19 import java.lang.management.ManagementFactory;
20 import java.time.LocalDateTime;
21 import java.util.concurrent.CountDownLatch;
22 import java.util.concurrent.atomic.AtomicInteger;
23
24 import org.apache.logging.log4j.LogManager;
25 import org.apache.logging.log4j.Logger;
26 import org.codelibs.core.lang.StringUtil;
27 import org.codelibs.core.misc.DynamicProperties;
28 import org.codelibs.core.timer.TimeoutManager;
29 import org.codelibs.core.timer.TimeoutTask;
30 import org.codelibs.fess.Constants;
31 import org.codelibs.fess.crawler.client.FesenClient;
32 import org.codelibs.fess.exception.ContainerNotAvailableException;
33 import org.codelibs.fess.helper.SuggestHelper;
34 import org.codelibs.fess.opensearch.client.SearchEngineClient;
35 import org.codelibs.fess.timer.LogNotificationTarget;
36 import org.codelibs.fess.timer.SystemMonitorTarget;
37 import org.codelibs.fess.util.ComponentUtil;
38 import org.codelibs.fess.util.SystemUtil;
39 import org.kohsuke.args4j.CmdLineException;
40 import org.kohsuke.args4j.CmdLineParser;
41 import org.kohsuke.args4j.Option;
42 import org.lastaflute.di.core.external.GenericExternalContext;
43 import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
44 import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
45 import org.opensearch.monitor.jvm.JvmInfo;
46 import org.opensearch.monitor.os.OsProbe;
47 import org.opensearch.monitor.process.ProcessProbe;
48
49 import jakarta.annotation.Resource;
50
51
52
53
54
55
56 public class SuggestCreator {
57
58
59
60
61 public SuggestCreator() {
62
63 }
64
65 private static final Logger logger = LogManager.getLogger(SuggestCreator.class);
66
67
68 @Resource
69 public SearchEngineClient searchEngineClient;
70
71
72
73
74 protected static class Options {
75
76 @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
77 protected String sessionId;
78
79
80 @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
81 protected String name;
82
83
84 @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
85 protected String propertiesPath;
86
87
88
89
90 protected Options() {
91
92 }
93
94 @Override
95 public String toString() {
96 return "Options [sessionId=" + sessionId + ", name=" + name + ", propertiesPath=" + propertiesPath + "]";
97 }
98 }
99
100
101
102
103 static void initializeProbes() {
104
105 ProcessProbe.getInstance();
106 OsProbe.getInstance();
107 JvmInfo.jvmInfo();
108 }
109
110
111
112
113
114
115 public static void main(final String[] args) {
116 final Options options = new Options();
117 final CmdLineParser parser = new CmdLineParser(options);
118 try {
119 parser.parseArgument(args);
120 } catch (final CmdLineException e) {
121 System.err.println(e.getMessage());
122 System.err.println("java " + SuggestCreator.class.getCanonicalName() + " [options...] arguments...");
123 parser.printUsage(System.err);
124 return;
125 }
126
127 if (logger.isDebugEnabled()) {
128 try {
129 ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
130 System.getProperties()
131 .entrySet()
132 .stream()
133 .forEach(e -> logger.debug("Property: {}={}", e.getKey(),
134 SystemUtil.maskSensitiveValue(String.valueOf(e.getKey()), String.valueOf(e.getValue()))));
135 System.getenv()
136 .entrySet()
137 .forEach(e -> logger.debug("Env: {}={}", e.getKey(), SystemUtil.maskSensitiveValue(e.getKey(), e.getValue())));
138 logger.debug("Options: options={}", options);
139 } catch (final Exception e) {
140
141 }
142 }
143
144 final String httpAddress = SystemUtil.getSearchEngineHttpAddress();
145 if (StringUtil.isNotBlank(httpAddress)) {
146 System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
147 }
148
149 TimeoutTask systemMonitorTask = null;
150 TimeoutTask logNotificationTask = null;
151 LogNotificationTarget logNotificationTarget = null;
152 int exitCode;
153 try {
154 SingletonLaContainerFactory.setConfigPath("app.xml");
155 SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
156 SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
157 SingletonLaContainerFactory.init();
158
159 final Thread shutdownCallback = new Thread("ShutdownHook") {
160 @Override
161 public void run() {
162 if (logger.isDebugEnabled()) {
163 logger.debug("Destroying LaContainer...");
164 }
165 destroyContainer();
166 }
167 };
168 Runtime.getRuntime().addShutdownHook(shutdownCallback);
169
170 systemMonitorTask = TimeoutManager.getInstance()
171 .addTimeoutTarget(new SystemMonitorTarget(), ComponentUtil.getFessConfig().getSuggestSystemMonitorIntervalAsInteger(),
172 true);
173
174 if (ComponentUtil.getFessConfig().isLogNotificationEnabled()) {
175 logNotificationTarget = new LogNotificationTarget();
176 logNotificationTask = TimeoutManager.getInstance()
177 .addTimeoutTarget(logNotificationTarget, ComponentUtil.getFessConfig().getLogNotificationFlushIntervalAsInteger(),
178 true);
179 }
180
181 exitCode = process(options);
182 } catch (final ContainerNotAvailableException e) {
183 if (logger.isDebugEnabled()) {
184 logger.debug("SuggestCreator is stopped.", e);
185 } else if (logger.isInfoEnabled()) {
186 logger.info("SuggestCreator is stopped.");
187 }
188 exitCode = Constants.EXIT_FAIL;
189 } catch (final Throwable t) {
190 logger.error("SuggestCreator terminated unexpectedly.", t);
191 exitCode = Constants.EXIT_FAIL;
192 } finally {
193 if (systemMonitorTask != null) {
194 systemMonitorTask.cancel();
195 }
196 if (logNotificationTask != null) {
197 logNotificationTask.cancel();
198 }
199 if (logNotificationTarget != null) {
200 logNotificationTarget.flush();
201 }
202 destroyContainer();
203 }
204
205 logger.info("Finished SuggestCreator.");
206 System.exit(exitCode);
207 }
208
209 private static void destroyContainer() {
210 TimeoutManager.getInstance().stop();
211 synchronized (SingletonLaContainerFactory.class) {
212 SingletonLaContainerFactory.destroy();
213 }
214 }
215
216 private static int process(final Options options) {
217 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
218
219 if (StringUtil.isNotBlank(options.propertiesPath)) {
220 systemProperties.reload(options.propertiesPath);
221 } else {
222 try {
223 final File propFile = ComponentUtil.getSystemHelper().createTempFile("suggest_", ".properties");
224 if (propFile.delete() && logger.isDebugEnabled()) {
225 logger.debug("Deleted temp file: path={}", propFile.getAbsolutePath());
226 }
227 systemProperties.reload(propFile.getAbsolutePath());
228 propFile.deleteOnExit();
229 } catch (final Exception e) {
230 logger.warn("Failed to create system properties file.", e);
231 }
232 }
233
234 final SuggestCreator creator = ComponentUtil.getComponent(SuggestCreator.class);
235 final LocalDateTime startTime = LocalDateTime.now();
236 int ret = creator.create();
237 if (ret == 0) {
238 ret = creator.purge(startTime);
239 }
240 return ret;
241 }
242
243 private int create() {
244 if (!ComponentUtil.getFessConfig().isSuggestDocuments() && !ComponentUtil.getFessConfig().isSuggestSearchLog()) {
245 logger.info("Skipped creating suggest index: both document and search log suggestions are disabled.");
246 return 0;
247 }
248
249 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
250
251 logger.info("Creating new suggest index.");
252 suggestHelper.suggester().createNextIndex();
253
254 logger.info("Storing all bad words.");
255 suggestHelper.storeAllBadWords(true);
256
257 logger.info("Storing all elevate words.");
258 suggestHelper.storeAllElevateWords(true);
259
260 final AtomicInteger exitCode = new AtomicInteger(0);
261
262 if (ComponentUtil.getFessConfig().isSuggestDocuments()) {
263 final CountDownLatch latch = new CountDownLatch(1);
264
265 logger.info("Parsing words from indexed documents.");
266 suggestHelper.indexFromDocuments(ret -> {
267 logger.info("Success indexing from documents.");
268 latch.countDown();
269 }, t -> {
270 logger.error("Failed to update suggest index.", t);
271 exitCode.set(1);
272 latch.countDown();
273 });
274
275 try {
276 latch.await();
277 } catch (final InterruptedException ignore) {
278 if (logger.isDebugEnabled()) {
279 logger.debug("Interrupted.", ignore);
280 }
281 exitCode.set(1);
282 }
283 }
284
285 if (ComponentUtil.getFessConfig().isSuggestSearchLog()) {
286 logger.info("Parsing words from search logs.");
287 try {
288 suggestHelper.storeSearchLog();
289 } catch (final Exception e) {
290 if (logger.isDebugEnabled()) {
291 logger.debug("Failed to update suggest index.", e);
292 }
293 exitCode.set(1);
294 }
295 }
296
297 logger.info("Replacing new suggest index.");
298 suggestHelper.suggester().switchIndex();
299
300 logger.info("Removing old indices.");
301 suggestHelper.suggester().removeDisableIndices();
302
303 return exitCode.get();
304 }
305
306 private int purge(final LocalDateTime time) {
307 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
308
309 try {
310 suggestHelper.purgeDocumentSuggest(time);
311 final long cleanupDay = ComponentUtil.getFessConfig().getPurgeSuggestSearchLogDay();
312 if (cleanupDay > 0) {
313 suggestHelper.purgeSearchlogSuggest(time.minusDays(cleanupDay));
314 }
315 return 0;
316 } catch (final Exception e) {
317 logger.warn("Failed to purge suggest data.", e);
318 return 1;
319 }
320 }
321
322 }