1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19 import static org.codelibs.core.stream.StreamUtil.stream;
20
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.UnsupportedEncodingException;
25 import java.net.InetAddress;
26 import java.net.URLEncoder;
27 import java.net.UnknownHostException;
28 import java.nio.file.Files;
29 import java.nio.file.Path;
30 import java.time.LocalDateTime;
31 import java.util.ArrayList;
32 import java.util.Calendar;
33 import java.util.Date;
34 import java.util.HashMap;
35 import java.util.LinkedHashMap;
36 import java.util.List;
37 import java.util.Locale;
38 import java.util.Map;
39 import java.util.Map.Entry;
40 import java.util.Properties;
41 import java.util.TimeZone;
42 import java.util.UUID;
43 import java.util.concurrent.ExecutionException;
44 import java.util.concurrent.TimeUnit;
45 import java.util.concurrent.atomic.AtomicBoolean;
46 import java.util.concurrent.atomic.AtomicInteger;
47 import java.util.regex.Pattern;
48 import java.util.stream.Collectors;
49
50 import javax.annotation.PostConstruct;
51 import javax.annotation.PreDestroy;
52 import javax.servlet.ServletContext;
53
54 import org.apache.commons.lang3.LocaleUtils;
55 import org.apache.commons.lang3.StringUtils;
56 import org.apache.logging.log4j.Level;
57 import org.apache.logging.log4j.LogManager;
58 import org.apache.logging.log4j.Logger;
59 import org.apache.logging.log4j.core.config.Configurator;
60 import org.codelibs.core.exception.IORuntimeException;
61 import org.codelibs.core.lang.StringUtil;
62 import org.codelibs.core.lang.ThreadUtil;
63 import org.codelibs.core.misc.Pair;
64 import org.codelibs.fesen.monitor.os.OsProbe;
65 import org.codelibs.fess.Constants;
66 import org.codelibs.fess.crawler.util.CharUtil;
67 import org.codelibs.fess.exception.FessSystemException;
68 import org.codelibs.fess.mylasta.action.FessMessages;
69 import org.codelibs.fess.mylasta.action.FessUserBean;
70 import org.codelibs.fess.mylasta.direction.FessConfig;
71 import org.codelibs.fess.util.ComponentUtil;
72 import org.codelibs.fess.util.GsaConfigParser;
73 import org.codelibs.fess.util.ParameterUtil;
74 import org.codelibs.fess.util.ResourceUtil;
75 import org.codelibs.fess.validation.FessActionValidator;
76 import org.lastaflute.core.message.supplier.UserMessagesCreator;
77 import org.lastaflute.web.TypicalAction;
78 import org.lastaflute.web.response.HtmlResponse;
79 import org.lastaflute.web.ruts.process.ActionRuntime;
80 import org.lastaflute.web.servlet.request.RequestManager;
81 import org.lastaflute.web.util.LaServletContextUtil;
82 import org.lastaflute.web.validation.ActionValidator;
83
84 import com.google.common.cache.CacheBuilder;
85 import com.google.common.cache.CacheLoader;
86 import com.google.common.cache.LoadingCache;
87 import com.ibm.icu.util.ULocale;
88
89 public class SystemHelper {
90
91 private static final Logger logger = LogManager.getLogger(SystemHelper.class);
92
93 protected final Map<String, String> designJspFileNameMap = new LinkedHashMap<>();
94
95 protected final AtomicBoolean forceStop = new AtomicBoolean(false);
96
97 protected LoadingCache<String, List<Map<String, String>>> langItemsCache;
98
99 protected String filterPathEncoding;
100
101 protected String[] supportedLanguages;
102
103 protected List<Runnable> shutdownHookList = new ArrayList<>();
104
105 protected AtomicInteger previousClusterState = new AtomicInteger(0);
106
107 protected String version;
108
109 protected int majorVersion;
110
111 protected int minorVersion;
112
113 protected String productVersion;
114
115 protected long eolTime;
116
117 private short systemCpuPercent;
118
119 private long systemCpuCheckTime;
120
121 private long systemCpuCheckInterval = 1000L;
122
123 @PostConstruct
124 public void init() {
125 if (logger.isDebugEnabled()) {
126 logger.debug("Initialize {}", this.getClass().getSimpleName());
127 }
128 final Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
129 cal.set(2023, 2 - 1, 3);
130 eolTime = cal.getTimeInMillis();
131 if (isEoled()) {
132 logger.error("Your system is out of support. See https://fess.codelibs.org/eol.html");
133 }
134 updateSystemProperties();
135 final FessConfig fessConfig = ComponentUtil.getFessConfig();
136 filterPathEncoding = fessConfig.getPathEncoding();
137 supportedLanguages = fessConfig.getSupportedLanguagesAsArray();
138 langItemsCache = CacheBuilder.newBuilder().maximumSize(20).expireAfterAccess(1, TimeUnit.HOURS)
139 .build(new CacheLoader<String, List<Map<String, String>>>() {
140 @Override
141 public List<Map<String, String>> load(final String key) throws Exception {
142 final ULocale uLocale = new ULocale(key);
143 final Locale displayLocale = uLocale.toLocale();
144 final List<Map<String, String>> langItems = new ArrayList<>(supportedLanguages.length);
145 final String msg = ComponentUtil.getMessageManager().getMessage(displayLocale, "labels.allLanguages");
146 final Map<String, String> defaultMap = new HashMap<>(2);
147 defaultMap.put(Constants.ITEM_LABEL, msg);
148 defaultMap.put(Constants.ITEM_VALUE, "all");
149 langItems.add(defaultMap);
150
151 for (final String lang : supportedLanguages) {
152 final Locale locale = LocaleUtils.toLocale(lang);
153 final String label = locale.getDisplayName(displayLocale);
154 final Map<String, String> map = new HashMap<>(2);
155 map.put(Constants.ITEM_LABEL, label);
156 map.put(Constants.ITEM_VALUE, lang);
157 langItems.add(map);
158 }
159 return langItems;
160 }
161 });
162
163 ComponentUtil.doInitProcesses(Runnable::run);
164
165 parseProjectProperties();
166 }
167
168 protected void parseProjectProperties() {
169 final Path propPath = ResourceUtil.getProjectPropertiesFile();
170 try (final InputStream in = Files.newInputStream(propPath)) {
171 final Properties prop = new Properties();
172 prop.load(in);
173 version = prop.getProperty("fess.version", "0.0.0");
174 final String[] values = version.split("\\.");
175 majorVersion = Integer.parseInt(values[0]);
176 minorVersion = Integer.parseInt(values[1]);
177 productVersion = majorVersion + "." + minorVersion;
178 System.setProperty("fess.version", version);
179 System.setProperty("fess.product.version", productVersion);
180 } catch (final Exception e) {
181 throw new FessSystemException("Failed to parse project.properties.", e);
182 }
183 }
184
185 @PreDestroy
186 public void destroy() {
187 shutdownHookList.forEach(action -> {
188 try {
189 action.run();
190 } catch (final Exception e) {
191 logger.warn("Failed to process shutdown task.", e);
192 }
193 });
194 }
195
196 public String getUsername() {
197 final RequestManager requestManager = ComponentUtil.getRequestManager();
198 return requestManager.findUserBean(FessUserBean.class).map(FessUserBean::getUserId).orElse(Constants.GUEST_USER);
199 }
200
201 public Date getCurrentTime() {
202 return new Date();
203 }
204
205 public long getCurrentTimeAsLong() {
206 return System.currentTimeMillis();
207 }
208
209 public LocalDateTime getCurrentTimeAsLocalDateTime() {
210 return LocalDateTime.now();
211 }
212
213 public String getLogFilePath() {
214 final String value = System.getProperty("fess.log.path");
215 if (value != null) {
216 return value;
217 }
218 final String userDir = System.getProperty("user.dir");
219 final File targetDir = new File(userDir, "target");
220 return new File(targetDir, "logs").getAbsolutePath();
221 }
222
223 public String encodeUrlFilter(final String path) {
224 if (filterPathEncoding == null || path == null) {
225 return path;
226 }
227
228 try {
229 final StringBuilder buf = new StringBuilder(path.length() + 100);
230 for (int i = 0; i < path.length(); i++) {
231 final char c = path.charAt(i);
232 if (CharUtil.isUrlChar(c) || c == '^' || c == '{' || c == '}' || c == '|' || c == '\\') {
233 buf.append(c);
234 } else {
235 buf.append(URLEncoder.encode(String.valueOf(c), filterPathEncoding));
236 }
237 }
238 return buf.toString();
239 } catch (final UnsupportedEncodingException e) {
240 return path;
241 }
242 }
243
244 public String normalizeConfigPath(final String path) {
245
246 if (StringUtil.isBlank(path)) {
247 return StringUtils.EMPTY;
248 }
249
250 final String p = path.trim();
251 if (p.startsWith("#")) {
252 return StringUtils.EMPTY;
253 }
254
255 if (p.startsWith(GsaConfigParser.CONTAINS)) {
256 return ".*" + Pattern.quote(p.substring(GsaConfigParser.CONTAINS.length())) + ".*";
257 }
258
259 if (p.startsWith(GsaConfigParser.REGEXP)) {
260 return p.substring(GsaConfigParser.REGEXP.length());
261 }
262
263 if (p.startsWith(GsaConfigParser.REGEXP_CASE)) {
264 return p.substring(GsaConfigParser.REGEXP_CASE.length());
265 }
266
267 if (p.startsWith(GsaConfigParser.REGEXP_IGNORE_CASE)) {
268 return "(?i)" + p.substring(GsaConfigParser.REGEXP_IGNORE_CASE.length());
269 }
270
271 return p;
272 }
273
274 public String getForumLink() {
275 final String url = ComponentUtil.getFessConfig().getForumLink();
276 if (StringUtil.isBlank(url)) {
277 return null;
278 }
279 String target = null;
280 final Locale locale = ComponentUtil.getRequestManager().getUserLocale();
281 if (locale != null) {
282 final String lang = locale.getLanguage();
283 if (ComponentUtil.getFessConfig().isOnlineHelpSupportedLang(lang)) {
284 target = lang.toUpperCase(Locale.ROOT);
285 }
286 }
287 return url.replaceFirst("\\{lang\\}", target == null ? "EN" : target);
288 }
289
290 public String getHelpLink(final String name) {
291 final String url = ComponentUtil.getFessConfig().getOnlineHelpBaseLink() + name + "-guide.html";
292 return getHelpUrl(url);
293 }
294
295 protected String getHelpUrl(final String url) {
296 final Locale locale = ComponentUtil.getRequestManager().getUserLocale();
297 if (locale != null) {
298 final String lang = locale.getLanguage();
299 if (ComponentUtil.getFessConfig().isOnlineHelpSupportedLang(lang)) {
300 return url.replaceFirst("\\{lang\\}", lang).replaceFirst("\\{version\\}", majorVersion + "." + minorVersion);
301 }
302 }
303 return getDefaultHelpLink(url);
304 }
305
306 protected String getDefaultHelpLink(final String url) {
307 return url.replaceFirst("/\\{lang\\}/", "/").replaceFirst("\\{version\\}", majorVersion + "." + minorVersion);
308 }
309
310 public void addDesignJspFileName(final String key, final String value) {
311 designJspFileNameMap.put(key, value);
312 }
313
314 public String getDesignJspFileName(final String fileName) {
315 return designJspFileNameMap.get(fileName);
316 }
317
318 @SuppressWarnings("unchecked")
319 public Pair<String, String>[] getDesignJspFileNames() {
320 return designJspFileNameMap.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue())).toArray(n -> new Pair[n]);
321 }
322
323 public void refreshDesignJspFiles() {
324 final ServletContext servletContext = LaServletContextUtil.getServletContext();
325 stream(ComponentUtil.getVirtualHostHelper().getVirtualHostPaths())
326 .of(stream -> stream.filter(s -> s != null && !"/".equals(s)).forEach(key -> {
327 designJspFileNameMap.entrySet().stream().forEach(e -> {
328 final File jspFile = new File(servletContext.getRealPath("/WEB-INF/view" + key + "/" + e.getValue()));
329 if (!jspFile.exists()) {
330 jspFile.getParentFile().mkdirs();
331 final File baseJspFile = new File(servletContext.getRealPath("/WEB-INF/view/" + e.getValue()));
332 try {
333 Files.copy(baseJspFile.toPath(), jspFile.toPath());
334 } catch (final IOException ex) {
335 logger.warn("Could not copy from {} to {}", baseJspFile.getAbsolutePath(), jspFile.getAbsolutePath(), ex);
336 }
337 }
338 });
339 }));
340 }
341
342 public boolean isForceStop() {
343 return forceStop.get();
344 }
345
346 public void setForceStop(final boolean b) {
347 forceStop.set(true);
348 }
349
350 public String generateDocId(final Map<String, Object> map) {
351 return UUID.randomUUID().toString().replace("-", StringUtil.EMPTY);
352 }
353
354 public String abbreviateLongText(final String str) {
355 return StringUtils.abbreviate(str, ComponentUtil.getFessConfig().getMaxLogOutputLengthAsInteger());
356 }
357
358 public String normalizeHtmlLang(final String value) {
359 final String defaultLang = ComponentUtil.getFessConfig().getCrawlerDocumentHtmlDefaultLang();
360 if (StringUtil.isNotBlank(defaultLang)) {
361 return defaultLang;
362 }
363
364 return normalizeLang(value);
365 }
366
367 public String normalizeLang(final String value) {
368 if (StringUtil.isBlank(value)) {
369 return null;
370 }
371
372 final String localeName = value.trim().toLowerCase(Locale.ENGLISH).replace("-", "_");
373
374 for (final String supportedLang : supportedLanguages) {
375 if (localeName.startsWith(supportedLang.toLowerCase(Locale.ENGLISH))) {
376 return supportedLang;
377 }
378 }
379 return null;
380 }
381
382 public List<Map<String, String>> getLanguageItems(final Locale locale) {
383 try {
384 final String localeStr = locale.toString();
385 return langItemsCache.get(localeStr);
386 } catch (final ExecutionException e) {
387 final List<Map<String, String>> langItems = new ArrayList<>(supportedLanguages.length);
388 final String msg = ComponentUtil.getMessageManager().getMessage(locale, "labels.allLanguages");
389 final Map<String, String> defaultMap = new HashMap<>(2);
390 defaultMap.put(Constants.ITEM_LABEL, msg);
391 defaultMap.put(Constants.ITEM_VALUE, "all");
392 langItems.add(defaultMap);
393 return langItems;
394 }
395 }
396
397 public void addShutdownHook(final Runnable hook) {
398 shutdownHookList.add(hook);
399 }
400
401 public String getHostname() {
402 final Map<String, String> env = getEnvMap();
403 if (env.containsKey("COMPUTERNAME")) {
404 return env.get("COMPUTERNAME");
405 }
406 if (env.containsKey("HOSTNAME")) {
407 return env.get("HOSTNAME");
408 }
409 try {
410 return InetAddress.getLocalHost().getHostAddress();
411 } catch (final UnknownHostException e) {
412 logger.debug("Unknown hostname.", e);
413 }
414 return "Unknown";
415 }
416
417 public void setupAdminHtmlData(final TypicalAction action, final ActionRuntime runtime) {
418 runtime.registerData("developmentMode", ComponentUtil.getSearchEngineClient().isEmbedded());
419 final FessConfig fessConfig = ComponentUtil.getFessConfig();
420 final String installationLink = fessConfig.getOnlineHelpInstallation();
421 runtime.registerData("installationLink", getHelpUrl(installationLink));
422 runtime.registerData("storageEnabled",
423 StringUtil.isNotBlank(fessConfig.getStorageEndpoint()) && StringUtil.isNotBlank(fessConfig.getStorageBucket()));
424 final boolean eoled = isEoled();
425 runtime.registerData("eoled", eoled);
426 if (eoled) {
427 final String eolLink = fessConfig.getOnlineHelpEol();
428 runtime.registerData("eolLink", getHelpUrl(eolLink));
429 }
430 }
431
432 public void setupSearchHtmlData(final TypicalAction action, final ActionRuntime runtime) {
433 runtime.registerData("developmentMode", ComponentUtil.getSearchEngineClient().isEmbedded());
434 final FessConfig fessConfig = ComponentUtil.getFessConfig();
435 final String installationLink = fessConfig.getOnlineHelpInstallation();
436 runtime.registerData("installationLink", getHelpUrl(installationLink));
437 final boolean eoled = isEoled();
438 runtime.registerData("eoled", eoled);
439 if (eoled) {
440 final String eolLink = fessConfig.getOnlineHelpEol();
441 runtime.registerData("eolLink", getHelpUrl(eolLink));
442 }
443 }
444
445 protected boolean isEoled() {
446 return getCurrentTimeAsLong() > eolTime;
447 }
448
449 public String getSearchRoleByUser(final String name) {
450 return createSearchRole(ComponentUtil.getFessConfig().getRoleSearchUserPrefix(), name);
451 }
452
453 public String getSearchRoleByGroup(final String name) {
454 return createSearchRole(ComponentUtil.getFessConfig().getRoleSearchGroupPrefix(), name);
455 }
456
457 public String getSearchRoleByRole(final String name) {
458 return createSearchRole(ComponentUtil.getFessConfig().getRoleSearchRolePrefix(), name);
459 }
460
461 protected String createSearchRole(final String type, final String name) {
462 final String value = type + ComponentUtil.getFessConfig().getCanonicalLdapName(name);
463 if (logger.isDebugEnabled()) {
464 logger.debug("Search Role: {}:{}={}", type, name, value);
465 }
466 return value;
467 }
468
469 public void reloadConfiguration() {
470 ComponentUtil.getSearchEngineClient().refresh();
471
472 ComponentUtil.getSuggestHelper().init();
473 ComponentUtil.getPopularWordHelper().init();
474
475 ComponentUtil.getLabelTypeHelper().update();
476 ComponentUtil.getPathMappingHelper().update();
477 ComponentUtil.getRelatedContentHelper().update();
478 ComponentUtil.getRelatedQueryHelper().update();
479 ComponentUtil.getKeyMatchHelper().update();
480
481 ComponentUtil.getLdapManager().updateConfig();
482 ComponentUtil.getJobManager().reboot();
483 updateSystemProperties();
484 }
485
486 public void updateSystemProperties() {
487 final String value = ComponentUtil.getFessConfig().getAppValue();
488 if (logger.isDebugEnabled()) {
489 logger.debug("system.properties: {}", value);
490 }
491 if (StringUtil.isNotBlank(value)) {
492 ParameterUtil.parse(ParameterUtil.encrypt(value)).entrySet().stream().filter(e -> {
493 final String key = e.getKey();
494 if (StringUtil.isBlank(key)) {
495 return false;
496 }
497 if (key.startsWith("fess.")) {
498 return true;
499 }
500 return System.getProperty(key) == null;
501 }).forEach(e -> {
502 if (logger.isDebugEnabled()) {
503 logger.debug("system.properties: setProperty({}, {})", e.getKey(), e.getValue());
504 }
505 System.setProperty(e.getKey(), e.getValue());
506 });
507 }
508 }
509
510 public String updateConfiguration() {
511 final StringBuilder buf = new StringBuilder();
512 buf.append("Label: ").append(ComponentUtil.getLabelTypeHelper().load()).append("\n");
513 buf.append("PathMapping: ").append(ComponentUtil.getPathMappingHelper().load()).append("\n");
514 buf.append("RelatedContent: ").append(ComponentUtil.getRelatedContentHelper().load()).append("\n");
515 buf.append("RelatedQuery: ").append(ComponentUtil.getRelatedQueryHelper().load()).append("\n");
516 buf.append("KeyMatch: ").append(ComponentUtil.getKeyMatchHelper().load()).append("\n");
517 return buf.toString();
518 }
519
520 public boolean isChangedClusterState(final int status) {
521 return previousClusterState.getAndSet(status) != status;
522 }
523
524 public ActionValidator<FessMessages> createValidator(final RequestManager requestManager,
525 final UserMessagesCreator<FessMessages> messagesCreator, final Class<?>[] runtimeGroups) {
526 return new FessActionValidator<>(requestManager, messagesCreator, runtimeGroups);
527 }
528
529 public HtmlResponse getRedirectResponseToLogin(final HtmlResponse response) {
530 return response;
531 }
532
533 public HtmlResponse getRedirectResponseToRoot(final HtmlResponse response) {
534 return response;
535 }
536
537 public void setLogLevel(final String level) {
538 final Level logLevel = Level.toLevel(level, Level.WARN);
539 System.setProperty(Constants.FESS_LOG_LEVEL, logLevel.toString());
540 split(ComponentUtil.getFessConfig().getLoggingAppPackages(), ",")
541 .of(stream -> stream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(s -> Configurator.setLevel(s, logLevel)));
542 }
543
544 public String getLogLevel() {
545 return System.getProperty(Constants.FESS_LOG_LEVEL, Level.WARN.toString());
546 }
547
548 public File createTempFile(final String prefix, final String suffix) {
549 try {
550 final File file = File.createTempFile(prefix, suffix);
551 if (logger.isDebugEnabled()) {
552 logger.debug("Create {} as a temp file.", file.getAbsolutePath());
553 }
554 return file;
555 } catch (final IOException e) {
556 throw new IORuntimeException(e);
557 }
558 }
559
560 public void calibrateCpuLoad() {
561 final int percent = ComponentUtil.getFessConfig().getAdaptiveLoadControlAsInteger();
562 if (percent <= 0) {
563 return;
564 }
565 while (getSystemCpuPercent() > percent) {
566 if (logger.isInfoEnabled()) {
567 logger.info("Cpu Load {}% is greater than {}%.", getSystemCpuPercent(), percent);
568 }
569 ThreadUtil.sleep(systemCpuCheckInterval);
570 }
571 }
572
573 protected short getSystemCpuPercent() {
574 final long now = System.currentTimeMillis();
575 if (now - systemCpuCheckTime > systemCpuCheckInterval) {
576 synchronized (this) {
577 if (now - systemCpuCheckTime > systemCpuCheckInterval) {
578 try {
579 final OsProbe osProbe = OsProbe.getInstance();
580 systemCpuPercent = osProbe.getSystemCpuPercent();
581 if (logger.isDebugEnabled()) {
582 logger.debug("Updated System Cpu {}%", systemCpuPercent);
583 }
584 } catch (final Exception e) {
585 logger.warn("Failed to get SystemCpuPercent.", e);
586 return 0;
587 }
588 systemCpuCheckTime = now;
589 }
590 }
591 }
592 return systemCpuPercent;
593 }
594
595 public Map<String, String> getFilteredEnvMap(final String keyPattern) {
596 final Pattern pattern = Pattern.compile(keyPattern);
597 return getEnvMap().entrySet().stream().filter(e -> {
598 final String key = e.getKey();
599 if (StringUtil.isBlank(key)) {
600 return false;
601 }
602 return pattern.matcher(key).matches();
603 }).collect(Collectors.toMap(Entry<String, String>::getKey, Entry<String, String>::getValue));
604 }
605
606 protected Map<String, String> getEnvMap() {
607 return System.getenv();
608 }
609
610 public String getVersion() {
611 return version;
612 }
613
614 public int getMajorVersion() {
615 return majorVersion;
616 }
617
618 public int getMinorVersion() {
619 return minorVersion;
620 }
621
622 public String getProductVersion() {
623 return productVersion;
624 }
625
626 public void setSystemCpuCheckInterval(final long systemCpuCheckInterval) {
627 this.systemCpuCheckInterval = systemCpuCheckInterval;
628 }
629 }