View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.helper;
17  
18  import java.util.Arrays;
19  import java.util.Map;
20  import java.util.stream.Collectors;
21  
22  import org.apache.logging.log4j.LogManager;
23  import org.apache.logging.log4j.Logger;
24  import org.apache.tika.language.detect.LanguageDetector;
25  import org.apache.tika.language.detect.LanguageResult;
26  import org.codelibs.core.lang.StringUtil;
27  import org.codelibs.fess.mylasta.direction.FessConfig;
28  import org.codelibs.fess.util.ComponentUtil;
29  import org.codelibs.fess.util.DocumentUtil;
30  import org.opensearch.script.Script;
31  
32  import jakarta.annotation.PostConstruct;
33  
34  /**
35   * Helper class for language detection.
36   */
37  public class LanguageHelper {
38      private static final Logger logger = LogManager.getLogger(LanguageHelper.class);
39  
40      /** An array of language fields. */
41      protected String[] langFields;
42  
43      /** An array of supported languages. */
44      protected String[] supportedLanguages;
45  
46      /** The language detector. */
47      protected LanguageDetector detector;
48  
49      /** The maximum text length for language detection. */
50      protected int maxTextLength;
51  
52      /**
53       * Default constructor.
54       */
55      public LanguageHelper() {
56          // do nothing
57      }
58  
59      /**
60       * Initializes the helper.
61       */
62      @PostConstruct
63      public void init() {
64          if (logger.isDebugEnabled()) {
65              logger.debug("Initializing {}", this.getClass().getSimpleName());
66          }
67          final FessConfig fessConfig = ComponentUtil.getFessConfig();
68          langFields = fessConfig.getIndexerLanguageFieldsAsArray();
69          supportedLanguages = fessConfig.getSupportedLanguagesAsArray();
70          maxTextLength = fessConfig.getIndexerLanguageDetectLengthAsInteger();
71      }
72  
73      /**
74       * Updates a document with language information.
75       *
76       * @param doc The document to update.
77       */
78      public void updateDocument(final Map<String, Object> doc) {
79          final FessConfig fessConfig = ComponentUtil.getFessConfig();
80          String language = getSupportedLanguage(DocumentUtil.getValue(doc, fessConfig.getIndexFieldLang(), String.class));
81          if (language == null) {
82              for (final String f : langFields) {
83                  if (doc.containsKey(f)) {
84                      language = detectLanguage(DocumentUtil.getValue(doc, f, String.class));
85                      if (language != null) {
86                          if (logger.isDebugEnabled()) {
87                              logger.debug("set {} to lang field", language);
88                          }
89                          doc.put(fessConfig.getIndexFieldLang(), language);
90                          break;
91                      }
92                  }
93              }
94              if (language == null) {
95                  return;
96              }
97          }
98  
99          for (final String f : langFields) {
100             final String lf = f + "_" + language;
101             if (doc.containsKey(f) && !doc.containsKey(lf)) {
102                 doc.put(lf, doc.get(f));
103                 if (logger.isDebugEnabled()) {
104                     logger.debug("add {} field", lf);
105                 }
106             }
107         }
108     }
109 
110     /**
111      * Detects the language of a text.
112      *
113      * @param text The text to detect the language from.
114      * @return The detected language.
115      */
116     protected String detectLanguage(final String text) {
117         if (StringUtil.isBlank(text)) {
118             return null;
119         }
120         final String target = getDetectText(text);
121         final LanguageResult result = detector.detect(target);
122         if (logger.isDebugEnabled()) {
123             logger.debug("detected lang:{}({}) from {}", result, result.getRawScore(), target);
124         }
125         return getSupportedLanguage(result.getLanguage());
126     }
127 
128     /**
129      * Returns the text to be used for language detection.
130      *
131      * @param text The original text.
132      * @return The text for language detection.
133      */
134     protected String getDetectText(final String text) {
135         final String result;
136         if (text.length() <= maxTextLength) {
137             result = text;
138         } else {
139             result = text.substring(0, maxTextLength);
140         }
141         return result.replaceAll("\\s+", " ");
142     }
143 
144     /**
145      * Returns the supported language for a given language.
146      *
147      * @param lang The language to check.
148      * @return The supported language, or null if not supported.
149      */
150     protected String getSupportedLanguage(final String lang) {
151         if (StringUtil.isBlank(lang)) {
152             return null;
153         }
154         for (final String l : supportedLanguages) {
155             if (l.equals(lang)) {
156                 return l;
157             }
158         }
159         return null;
160     }
161 
162     /**
163      * Sets the language detector.
164      *
165      * @param detector The language detector.
166      */
167     public void setDetector(final LanguageDetector detector) {
168         this.detector = detector;
169     }
170 
171     /**
172      * Creates a script for updating a document with language information.
173      *
174      * @param doc The document.
175      * @param code The script code.
176      * @return The script.
177      */
178     public Script createScript(final Map<String, Object> doc, final String code) {
179         final StringBuilder buf = new StringBuilder(100);
180         buf.append(code);
181         final FessConfig fessConfig = ComponentUtil.getFessConfig();
182         final String language = DocumentUtil.getValue(doc, fessConfig.getIndexFieldLang(), String.class);
183         if (StringUtil.isNotBlank(language)) {
184             for (final String f : langFields) {
185                 buf.append(";ctx._source.").append(f).append('_').append(language).append("=ctx._source.").append(f);
186             }
187         }
188         if (logger.isDebugEnabled()) {
189             logger.debug("update script: {}", buf);
190         }
191         return new Script(buf.toString());
192     }
193 
194     /**
195      * Returns the reindex script source.
196      *
197      * @return The reindex script source.
198      */
199     public String getReindexScriptSource() {
200         final FessConfig fessConfig = ComponentUtil.getFessConfig();
201         final String langField = fessConfig.getIndexFieldLang();
202         final String code = Arrays.stream(langFields)
203                 .map(s -> "ctx._source['" + s + "_'+ctx._source." + langField + "]=ctx._source." + s)
204                 .collect(Collectors.joining(";"));
205         if (logger.isDebugEnabled()) {
206             logger.debug("reindex script: {}", code);
207         }
208         return "if(ctx._source." + langField + "!=null){" + code + "}";
209     }
210 
211 }