1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.exec;
17
18 import java.io.File;
19 import java.io.IOException;
20 import java.lang.management.ManagementFactory;
21 import java.time.LocalDateTime;
22 import java.util.concurrent.CountDownLatch;
23 import java.util.concurrent.atomic.AtomicInteger;
24
25 import javax.annotation.Resource;
26
27 import org.codelibs.core.lang.StringUtil;
28 import org.codelibs.core.misc.DynamicProperties;
29 import org.codelibs.fess.Constants;
30 import org.codelibs.fess.crawler.client.EsClient;
31 import org.codelibs.fess.es.client.FessEsClient;
32 import org.codelibs.fess.exception.ContainerNotAvailableException;
33 import org.codelibs.fess.helper.SuggestHelper;
34 import org.codelibs.fess.util.ComponentUtil;
35 import org.kohsuke.args4j.CmdLineException;
36 import org.kohsuke.args4j.CmdLineParser;
37 import org.kohsuke.args4j.Option;
38 import org.lastaflute.di.core.external.GenericExternalContext;
39 import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
40 import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43
44 public class SuggestCreator {
45
46 private static final Logger logger = LoggerFactory.getLogger(SuggestCreator.class);
47
48 @Resource
49 public FessEsClient fessEsClient;
50
51 protected static class Options {
52 @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
53 protected String sessionId;
54
55 @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
56 protected String name;
57
58 @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
59 protected String propertiesPath;
60
61 protected Options() {
62
63 }
64
65 @Override
66 public String toString() {
67 return "Options [sessionId=" + sessionId + ", name=" + name + ", propertiesPath=" + propertiesPath + "]";
68 }
69 }
70
71 public static void main(final String[] args) {
72 final Options options = new Options();
73
74 final CmdLineParser parser = new CmdLineParser(options);
75 try {
76 parser.parseArgument(args);
77 } catch (final CmdLineException e) {
78 System.err.println(e.getMessage());
79 System.err.println("java " + Crawler.class.getCanonicalName() + " [options...] arguments...");
80 parser.printUsage(System.err);
81 return;
82 }
83
84 if (logger.isDebugEnabled()) {
85 try {
86 ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: " + s));
87 System.getProperties().entrySet().stream().forEach(e -> logger.debug("Property: " + e.getKey() + "=" + e.getValue()));
88 System.getenv().entrySet().forEach(e -> logger.debug("Env: " + e.getKey() + "=" + e.getValue()));
89 logger.debug("Option: " + options);
90 } catch (final Exception e) {
91
92 }
93 }
94
95 final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
96 if (StringUtil.isNotBlank(transportAddresses)) {
97 System.setProperty(EsClient.TRANSPORT_ADDRESSES, transportAddresses);
98 }
99 final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
100 if (StringUtil.isNotBlank(clusterName)) {
101 System.setProperty(EsClient.CLUSTER_NAME, clusterName);
102 }
103
104 int exitCode;
105 try {
106 SingletonLaContainerFactory.setConfigPath("app.xml");
107 SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
108 SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
109 SingletonLaContainerFactory.init();
110
111 final Thread shutdownCallback = new Thread("ShutdownHook") {
112 @Override
113 public void run() {
114 if (logger.isDebugEnabled()) {
115 logger.debug("Destroying LaContainer..");
116 }
117 destroyContainer();
118 }
119 };
120 Runtime.getRuntime().addShutdownHook(shutdownCallback);
121 exitCode = process(options);
122 } catch (final ContainerNotAvailableException e) {
123 if (logger.isDebugEnabled()) {
124 logger.debug("Crawler is stopped.", e);
125 } else if (logger.isInfoEnabled()) {
126 logger.info("Crawler is stopped.");
127 }
128 exitCode = Constants.EXIT_FAIL;
129 } catch (final Throwable t) {
130 logger.error("Suggest creator does not work correctly.", t);
131 exitCode = Constants.EXIT_FAIL;
132 } finally {
133 destroyContainer();
134 }
135
136 logger.info("Finished suggestCreator.");
137 System.exit(exitCode);
138 }
139
140 private static void destroyContainer() {
141 synchronized (SingletonLaContainerFactory.class) {
142 SingletonLaContainerFactory.destroy();
143 }
144 }
145
146 private static int process(final Options options) {
147 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
148
149 if (StringUtil.isNotBlank(options.propertiesPath)) {
150 systemProperties.reload(options.propertiesPath);
151 } else {
152 try {
153 final File propFile = File.createTempFile("suggest_", ".properties");
154 if (propFile.delete() && logger.isDebugEnabled()) {
155 logger.debug("Deleted a temp file: " + propFile.getAbsolutePath());
156 }
157 systemProperties.reload(propFile.getAbsolutePath());
158 propFile.deleteOnExit();
159 } catch (final IOException e) {
160 logger.warn("Failed to create system properties file.", e);
161 }
162 }
163
164 final SuggestCreator creator = ComponentUtil.getComponent(SuggestCreator.class);
165 final LocalDateTime startTime = LocalDateTime.now();
166 int ret = creator.create();
167 if (ret == 0) {
168 ret = creator.purge(startTime);
169 }
170 return ret;
171 }
172
173 private int create() {
174 if (!ComponentUtil.getFessConfig().isSuggestDocuments()) {
175 logger.info("Skip create suggest document.");
176 return 0;
177 }
178
179 logger.info("Start create suggest document.");
180
181 final AtomicInteger result = new AtomicInteger(1);
182 final CountDownLatch latch = new CountDownLatch(1);
183
184 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
185
186 logger.info("Create update index.");
187 suggestHelper.suggester().createNextIndex();
188
189 logger.info("storeAllBadWords");
190 suggestHelper.storeAllBadWords(true);
191
192 logger.info("storeAllElevateWords");
193 suggestHelper.storeAllElevateWords(true);
194
195 logger.info("indexFromDocuments");
196 suggestHelper.indexFromDocuments(ret -> {
197 logger.info("Success index from documents.");
198 result.set(0);
199 latch.countDown();
200 }, t -> {
201 logger.error("Failed to update suggest index.", t);
202 latch.countDown();
203 });
204
205 try {
206 latch.await();
207 } catch (final InterruptedException ignore) {
208 if (logger.isDebugEnabled()) {
209 logger.debug("Interrupted.", ignore);
210 }
211 }
212
213 logger.info("storeSearchLog");
214 suggestHelper.storeSearchLog();
215
216 logger.info("switchIndex");
217 suggestHelper.suggester().switchIndex();
218
219 logger.info("removeDisableIndices");
220 suggestHelper.suggester().removeDisableIndices();
221
222 return result.get();
223 }
224
225 private int purge(final LocalDateTime time) {
226 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
227
228 try {
229 suggestHelper.purgeDocumentSuggest(time);
230 final long cleanupDay = ComponentUtil.getFessConfig().getPurgeSuggestSearchLogDay();
231 if (cleanupDay > 0) {
232 suggestHelper.purgeSearchlogSuggest(time.minusDays(cleanupDay));
233 }
234 return 0;
235 } catch (final Exception e) {
236 logger.info("Purge error.", e);
237 return 1;
238 }
239 }
240
241 }