View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.exec;
17  
18  import java.io.File;
19  import java.lang.management.ManagementFactory;
20  import java.time.LocalDateTime;
21  import java.util.concurrent.CountDownLatch;
22  import java.util.concurrent.atomic.AtomicInteger;
23  
24  import org.apache.logging.log4j.LogManager;
25  import org.apache.logging.log4j.Logger;
26  import org.codelibs.core.lang.StringUtil;
27  import org.codelibs.core.misc.DynamicProperties;
28  import org.codelibs.core.timer.TimeoutManager;
29  import org.codelibs.core.timer.TimeoutTask;
30  import org.codelibs.fess.Constants;
31  import org.codelibs.fess.crawler.client.FesenClient;
32  import org.codelibs.fess.exception.ContainerNotAvailableException;
33  import org.codelibs.fess.helper.SuggestHelper;
34  import org.codelibs.fess.opensearch.client.SearchEngineClient;
35  import org.codelibs.fess.timer.LogNotificationTarget;
36  import org.codelibs.fess.timer.SystemMonitorTarget;
37  import org.codelibs.fess.util.ComponentUtil;
38  import org.codelibs.fess.util.SystemUtil;
39  import org.kohsuke.args4j.CmdLineException;
40  import org.kohsuke.args4j.CmdLineParser;
41  import org.kohsuke.args4j.Option;
42  import org.lastaflute.di.core.external.GenericExternalContext;
43  import org.lastaflute.di.core.external.GenericExternalContextComponentDefRegister;
44  import org.lastaflute.di.core.factory.SingletonLaContainerFactory;
45  import org.opensearch.monitor.jvm.JvmInfo;
46  import org.opensearch.monitor.os.OsProbe;
47  import org.opensearch.monitor.process.ProcessProbe;
48  
49  import jakarta.annotation.Resource;
50  
51  /**
52   * This class is a command-line application for creating and managing the suggest index.
53   * It provides functionality to index words from documents and search logs, as well as
54   * to purge old suggest data.
55   */
56  public class SuggestCreator {
57  
58      /**
59       * Constructs a new suggest creator.
60       */
61      public SuggestCreator() {
62          // do nothing
63      }
64  
65      private static final Logger logger = LogManager.getLogger(SuggestCreator.class);
66  
67      /** The search engine client for interacting with OpenSearch. */
68      @Resource
69      public SearchEngineClient searchEngineClient;
70  
71      /**
72       * A nested class for parsing command-line options.
73       */
74      protected static class Options {
75          /** The session ID for the suggest creation process. */
76          @Option(name = "-s", aliases = "--sessionId", metaVar = "sessionId", usage = "Session ID")
77          protected String sessionId;
78  
79          /** The name of the suggest creator instance. */
80          @Option(name = "-n", aliases = "--name", metaVar = "name", usage = "Name")
81          protected String name;
82  
83          /** The path to the properties file for configuration. */
84          @Option(name = "-p", aliases = "--properties", metaVar = "properties", usage = "Properties File")
85          protected String propertiesPath;
86  
87          /**
88           * Constructs a new Options object.
89           */
90          protected Options() {
91              // nothing
92          }
93  
94          @Override
95          public String toString() {
96              return "Options [sessionId=" + sessionId + ", name=" + name + ", propertiesPath=" + propertiesPath + "]";
97          }
98      }
99  
100     /**
101      * Initializes the necessary probes for monitoring system resources.
102      */
103     static void initializeProbes() {
104         // Force probes to be loaded
105         ProcessProbe.getInstance();
106         OsProbe.getInstance();
107         JvmInfo.jvmInfo();
108     }
109 
110     /**
111      * The main method for the SuggestCreator application.
112      *
113      * @param args The command-line arguments.
114      */
115     public static void main(final String[] args) {
116         final Options options = new Options();
117         final CmdLineParser parser = new CmdLineParser(options);
118         try {
119             parser.parseArgument(args);
120         } catch (final CmdLineException e) {
121             System.err.println(e.getMessage());
122             System.err.println("java " + SuggestCreator.class.getCanonicalName() + " [options...] arguments...");
123             parser.printUsage(System.err);
124             return;
125         }
126 
127         if (logger.isDebugEnabled()) {
128             try {
129                 ManagementFactory.getRuntimeMXBean().getInputArguments().stream().forEach(s -> logger.debug("Parameter: {}", s));
130                 System.getProperties()
131                         .entrySet()
132                         .stream()
133                         .forEach(e -> logger.debug("Property: {}={}", e.getKey(),
134                                 SystemUtil.maskSensitiveValue(String.valueOf(e.getKey()), String.valueOf(e.getValue()))));
135                 System.getenv()
136                         .entrySet()
137                         .forEach(e -> logger.debug("Env: {}={}", e.getKey(), SystemUtil.maskSensitiveValue(e.getKey(), e.getValue())));
138                 logger.debug("Options: options={}", options);
139             } catch (final Exception e) {
140                 // ignore
141             }
142         }
143 
144         final String httpAddress = SystemUtil.getSearchEngineHttpAddress();
145         if (StringUtil.isNotBlank(httpAddress)) {
146             System.setProperty(FesenClient.HTTP_ADDRESS, httpAddress);
147         }
148 
149         TimeoutTask systemMonitorTask = null;
150         TimeoutTask logNotificationTask = null;
151         LogNotificationTarget logNotificationTarget = null;
152         int exitCode;
153         try {
154             SingletonLaContainerFactory.setConfigPath("app.xml");
155             SingletonLaContainerFactory.setExternalContext(new GenericExternalContext());
156             SingletonLaContainerFactory.setExternalContextComponentDefRegister(new GenericExternalContextComponentDefRegister());
157             SingletonLaContainerFactory.init();
158 
159             final Thread shutdownCallback = new Thread("ShutdownHook") {
160                 @Override
161                 public void run() {
162                     if (logger.isDebugEnabled()) {
163                         logger.debug("Destroying LaContainer...");
164                     }
165                     destroyContainer();
166                 }
167             };
168             Runtime.getRuntime().addShutdownHook(shutdownCallback);
169 
170             systemMonitorTask = TimeoutManager.getInstance()
171                     .addTimeoutTarget(new SystemMonitorTarget(), ComponentUtil.getFessConfig().getSuggestSystemMonitorIntervalAsInteger(),
172                             true);
173 
174             if (ComponentUtil.getFessConfig().isLogNotificationEnabled()) {
175                 logNotificationTarget = new LogNotificationTarget();
176                 logNotificationTask = TimeoutManager.getInstance()
177                         .addTimeoutTarget(logNotificationTarget, ComponentUtil.getFessConfig().getLogNotificationFlushIntervalAsInteger(),
178                                 true);
179             }
180 
181             exitCode = process(options);
182         } catch (final ContainerNotAvailableException e) {
183             if (logger.isDebugEnabled()) {
184                 logger.debug("SuggestCreator is stopped.", e);
185             } else if (logger.isInfoEnabled()) {
186                 logger.info("SuggestCreator is stopped.");
187             }
188             exitCode = Constants.EXIT_FAIL;
189         } catch (final Throwable t) {
190             logger.error("SuggestCreator terminated unexpectedly.", t);
191             exitCode = Constants.EXIT_FAIL;
192         } finally {
193             if (systemMonitorTask != null) {
194                 systemMonitorTask.cancel();
195             }
196             if (logNotificationTask != null) {
197                 logNotificationTask.cancel();
198             }
199             if (logNotificationTarget != null) {
200                 logNotificationTarget.flush();
201             }
202             destroyContainer();
203         }
204 
205         logger.info("Finished SuggestCreator.");
206         System.exit(exitCode);
207     }
208 
209     private static void destroyContainer() {
210         TimeoutManager.getInstance().stop();
211         synchronized (SingletonLaContainerFactory.class) {
212             SingletonLaContainerFactory.destroy();
213         }
214     }
215 
216     private static int process(final Options options) {
217         final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
218 
219         if (StringUtil.isNotBlank(options.propertiesPath)) {
220             systemProperties.reload(options.propertiesPath);
221         } else {
222             try {
223                 final File propFile = ComponentUtil.getSystemHelper().createTempFile("suggest_", ".properties");
224                 if (propFile.delete() && logger.isDebugEnabled()) {
225                     logger.debug("Deleted temp file: path={}", propFile.getAbsolutePath());
226                 }
227                 systemProperties.reload(propFile.getAbsolutePath());
228                 propFile.deleteOnExit();
229             } catch (final Exception e) {
230                 logger.warn("Failed to create system properties file.", e);
231             }
232         }
233 
234         final SuggestCreator creator = ComponentUtil.getComponent(SuggestCreator.class);
235         final LocalDateTime startTime = LocalDateTime.now();
236         int ret = creator.create();
237         if (ret == 0) {
238             ret = creator.purge(startTime);
239         }
240         return ret;
241     }
242 
243     private int create() {
244         if (!ComponentUtil.getFessConfig().isSuggestDocuments() && !ComponentUtil.getFessConfig().isSuggestSearchLog()) {
245             logger.info("Skipped creating suggest index: both document and search log suggestions are disabled.");
246             return 0;
247         }
248 
249         final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
250 
251         logger.info("Creating new suggest index.");
252         suggestHelper.suggester().createNextIndex();
253 
254         logger.info("Storing all bad words.");
255         suggestHelper.storeAllBadWords(true);
256 
257         logger.info("Storing all elevate words.");
258         suggestHelper.storeAllElevateWords(true);
259 
260         final AtomicInteger exitCode = new AtomicInteger(0);
261 
262         if (ComponentUtil.getFessConfig().isSuggestDocuments()) {
263             final CountDownLatch latch = new CountDownLatch(1);
264 
265             logger.info("Parsing words from indexed documents.");
266             suggestHelper.indexFromDocuments(ret -> {
267                 logger.info("Success indexing from documents.");
268                 latch.countDown();
269             }, t -> {
270                 logger.error("Failed to update suggest index.", t);
271                 exitCode.set(1);
272                 latch.countDown();
273             });
274 
275             try {
276                 latch.await();
277             } catch (final InterruptedException ignore) {
278                 if (logger.isDebugEnabled()) {
279                     logger.debug("Interrupted.", ignore);
280                 }
281                 exitCode.set(1);
282             }
283         }
284 
285         if (ComponentUtil.getFessConfig().isSuggestSearchLog()) {
286             logger.info("Parsing words from search logs.");
287             try {
288                 suggestHelper.storeSearchLog();
289             } catch (final Exception e) {
290                 if (logger.isDebugEnabled()) {
291                     logger.debug("Failed to update suggest index.", e);
292                 }
293                 exitCode.set(1);
294             }
295         }
296 
297         logger.info("Replacing new suggest index.");
298         suggestHelper.suggester().switchIndex();
299 
300         logger.info("Removing old indices.");
301         suggestHelper.suggester().removeDisableIndices();
302 
303         return exitCode.get();
304     }
305 
306     private int purge(final LocalDateTime time) {
307         final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
308 
309         try {
310             suggestHelper.purgeDocumentSuggest(time);
311             final long cleanupDay = ComponentUtil.getFessConfig().getPurgeSuggestSearchLogDay();
312             if (cleanupDay > 0) {
313                 suggestHelper.purgeSearchlogSuggest(time.minusDays(cleanupDay));
314             }
315             return 0;
316         } catch (final Exception e) {
317             logger.warn("Failed to purge suggest data.", e);
318             return 1;
319         }
320     }
321 
322 }