1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.exec;
17
18 import java.io.File;
19 import java.lang.management.ManagementFactory;
20 import java.time.LocalDateTime;
21 import java.util.concurrent.CountDownLatch;
22 import java.util.concurrent.atomic.AtomicInteger;
23
24 import javax.annotation.Resource;
25
26 import org.apache.logging.log4j.LogManager;
27 import org.apache.logging.log4j.Logger;
28 import org.codelibs.core.lang.StringUtil;
29 import org.codelibs.core.misc.DynamicProperties;
30 import org.codelibs.core.timer.TimeoutManager;
31 import org.codelibs.core.timer.TimeoutTask;
32 import org.codelibs.fesen.monitor.jvm.JvmInfo;
33 import org.codelibs.fesen.monitor.os.OsProbe;
34 import org.codelibs.fesen.monitor.process.ProcessProbe;
35 import org.codelibs.fess.Constants;
36 import org.codelibs.fess.crawler.client.FesenClient;
37 import org.codelibs.fess.es.client.SearchEngineClient;
38 import org.codelibs.fess.exception.ContainerNotAvailableException;
39 import org.codelibs.fess.helper.SuggestHelper;
40 import org.codelibs.fess.timer.SystemMonitorTarget;
41 import org.codelibs.fess.util.ComponentUtil;
42 import org.kohsuke.args4j.CmdLineException;
43 import org.kohsuke.args4j.CmdLineParser;
44 import org.kohsuke.args4j.Option;
45 import org.lastaflute.di.core.external.GenericExternalContext;
46 import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
47 import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
48
49 public class SuggestCreator {
50
51 private static final Logger logger = LogManager.getLogger(SuggestCreator.class);
52
53 @Resource
54 public SearchEngineClient searchEngineClient;
55
56 protected static class Options {
57 @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
58 protected String sessionId;
59
60 @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
61 protected String name;
62
63 @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
64 protected String propertiesPath;
65
66 protected Options() {
67
68 }
69
70 @Override
71 public String toString() {
72 return "Options [sessionId=" + sessionId + ", name=" + name + ", propertiesPath=" + propertiesPath + "]";
73 }
74 }
75
76 static void initializeProbes() {
77
78 ProcessProbe.getInstance();
79 OsProbe.getInstance();
80 JvmInfo.jvmInfo();
81 }
82
83 public static void main(final String[] args) {
84 final Options options = new Options();
85 final CmdLineParser parser = new CmdLineParser(options);
86 try {
87 parser.parseArgument(args);
88 } catch (final CmdLineException e) {
89 System.err.println(e.getMessage());
90 System.err.println("java " + SuggestCreator.class.getCanonicalName() + " [options...] arguments...");
91 parser.printUsage(System.err);
92 return;
93 }
94
95 if (logger.isDebugEnabled()) {
96 try {
97 ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
98 System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: {}={}", e.getKey(), e.getValue()));
99 System.getenv().entrySet().forEach(e -> logger.debug("Env: {}={}", e.getKey(), e.getValue()));
100 logger.debug("Option: {}", options);
101 } catch (final Exception e) {
102
103 }
104 }
105
106 final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
107 if (StringUtil.isNotBlank(httpAddress)) {
108 System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
109 }
110
111 TimeoutTask systemMonitorTask = null;
112 int exitCode;
113 try {
114 SingletonLaContainerFactory.setConfigPath("app.xml");
115 SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
116 SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
117 SingletonLaContainerFactory.init();
118
119 final Thread shutdownCallback = new Thread("ShutdownHook") {
120 @Override
121 public void run() {
122 if (logger.isDebugEnabled()) {
123 logger.debug("Destroying LaContainer..");
124 }
125 destroyContainer();
126 }
127 };
128 Runtime.getRuntime().addShutdownHook(shutdownCallback);
129
130 systemMonitorTask = TimeoutManager.getInstance().addTimeoutTarget(new SystemMonitorTarget(),
131 ComponentUtil.getFessConfig().getSuggestSystemMonitorIntervalAsInteger(), true);
132
133 exitCode = process(options);
134 } catch (final ContainerNotAvailableException e) {
135 if (logger.isDebugEnabled()) {
136 logger.debug("SuggestCreator is stopped.", e);
137 } else if (logger.isInfoEnabled()) {
138 logger.info("SuggestCreator is stopped.");
139 }
140 exitCode = Constants.EXIT_FAIL;
141 } catch (final Throwable t) {
142 logger.error("Suggest creator does not work correctly.", t);
143 exitCode = Constants.EXIT_FAIL;
144 } finally {
145 if (systemMonitorTask != null) {
146 systemMonitorTask.cancel();
147 }
148 destroyContainer();
149 }
150
151 logger.info("Finished SuggestCreator.");
152 System.exit(exitCode);
153 }
154
155 private static void destroyContainer() {
156 TimeoutManager.getInstance().stop();
157 synchronized (SingletonLaContainerFactory.class) {
158 SingletonLaContainerFactory.destroy();
159 }
160 }
161
162 private static int process(final Options options) {
163 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
164
165 if (StringUtil.isNotBlank(options.propertiesPath)) {
166 systemProperties.reload(options.propertiesPath);
167 } else {
168 try {
169 final File propFile = ComponentUtil.getSystemHelper().createTempFile("suggest_", ".properties");
170 if (propFile.delete() && logger.isDebugEnabled()) {
171 logger.debug("Deleted a temp file: {}", propFile.getAbsolutePath());
172 }
173 systemProperties.reload(propFile.getAbsolutePath());
174 propFile.deleteOnExit();
175 } catch (final Exception e) {
176 logger.warn("Failed to create system properties file.", e);
177 }
178 }
179
180 final SuggestCreator creator = ComponentUtil.getComponent(SuggestCreator.class);
181 final LocalDateTime startTime = LocalDateTime.now();
182 int ret = creator.create();
183 if (ret == 0) {
184 ret = creator.purge(startTime);
185 }
186 return ret;
187 }
188
189 private int create() {
190 if (!ComponentUtil.getFessConfig().isSuggestDocuments()) {
191 logger.info("Skip create suggest document.");
192 return 0;
193 }
194
195 logger.info("Start create suggest document.");
196
197 final AtomicInteger result = new AtomicInteger(1);
198 final CountDownLatch latch = new CountDownLatch(1);
199
200 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
201
202 logger.info("Create update index.");
203 suggestHelper.suggester().createNextIndex();
204
205 logger.info("Store all bad words.");
206 suggestHelper.storeAllBadWords(true);
207
208 logger.info("Store all elevate words.");
209 suggestHelper.storeAllElevateWords(true);
210
211 logger.info("Parse words from indexed documents.");
212 suggestHelper.indexFromDocuments(ret -> {
213 logger.info("Success index from documents.");
214 result.set(0);
215 latch.countDown();
216 }, t -> {
217 logger.error("Failed to update suggest index.", t);
218 latch.countDown();
219 });
220
221 try {
222 latch.await();
223 } catch (final InterruptedException ignore) {
224 if (logger.isDebugEnabled()) {
225 logger.debug("Interrupted.", ignore);
226 }
227 }
228
229 logger.info("Store search logs.");
230 suggestHelper.storeSearchLog();
231
232 logger.info("Switch indices.");
233 suggestHelper.suggester().switchIndex();
234
235 logger.info("Remove old indices.");
236 suggestHelper.suggester().removeDisableIndices();
237
238 return result.get();
239 }
240
241 private int purge(final LocalDateTime time) {
242 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
243
244 try {
245 suggestHelper.purgeDocumentSuggest(time);
246 final long cleanupDay = ComponentUtil.getFessConfig().getPurgeSuggestSearchLogDay();
247 if (cleanupDay > 0) {
248 suggestHelper.purgeSearchlogSuggest(time.minusDays(cleanupDay));
249 }
250 return 0;
251 } catch (final Exception e) {
252 logger.info("Purge error.", e);
253 return 1;
254 }
255 }
256
257 }