1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import java.util.Arrays;
19 import java.util.Map;
20 import java.util.stream.Collectors;
21
22 import org.apache.logging.log4j.LogManager;
23 import org.apache.logging.log4j.Logger;
24 import org.apache.tika.language.detect.LanguageDetector;
25 import org.apache.tika.language.detect.LanguageResult;
26 import org.codelibs.core.lang.StringUtil;
27 import org.codelibs.fess.mylasta.direction.FessConfig;
28 import org.codelibs.fess.util.ComponentUtil;
29 import org.codelibs.fess.util.DocumentUtil;
30 import org.opensearch.script.Script;
31
32 import jakarta.annotation.PostConstruct;
33
34
35
36
37 public class LanguageHelper {
38 private static final Logger logger = LogManager.getLogger(LanguageHelper.class);
39
40
41 protected String[] langFields;
42
43
44 protected String[] supportedLanguages;
45
46
47 protected LanguageDetector detector;
48
49
50 protected int maxTextLength;
51
52
53
54
55 public LanguageHelper() {
56
57 }
58
59
60
61
62 @PostConstruct
63 public void init() {
64 if (logger.isDebugEnabled()) {
65 logger.debug("Initializing {}", this.getClass().getSimpleName());
66 }
67 final FessConfig fessConfig = ComponentUtil.getFessConfig();
68 langFields = fessConfig.getIndexerLanguageFieldsAsArray();
69 supportedLanguages = fessConfig.getSupportedLanguagesAsArray();
70 maxTextLength = fessConfig.getIndexerLanguageDetectLengthAsInteger();
71 }
72
73
74
75
76
77
78 public void updateDocument(final Map<String, Object> doc) {
79 final FessConfig fessConfig = ComponentUtil.getFessConfig();
80 String language = getSupportedLanguage(DocumentUtil.getValue(doc, fessConfig.getIndexFieldLang(), String.class));
81 if (language == null) {
82 for (final String f : langFields) {
83 if (doc.containsKey(f)) {
84 language = detectLanguage(DocumentUtil.getValue(doc, f, String.class));
85 if (language != null) {
86 if (logger.isDebugEnabled()) {
87 logger.debug("set {} to lang field", language);
88 }
89 doc.put(fessConfig.getIndexFieldLang(), language);
90 break;
91 }
92 }
93 }
94 if (language == null) {
95 return;
96 }
97 }
98
99 for (final String f : langFields) {
100 final String lf = f + "_" + language;
101 if (doc.containsKey(f) && !doc.containsKey(lf)) {
102 doc.put(lf, doc.get(f));
103 if (logger.isDebugEnabled()) {
104 logger.debug("add {} field", lf);
105 }
106 }
107 }
108 }
109
110
111
112
113
114
115
116 protected String detectLanguage(final String text) {
117 if (StringUtil.isBlank(text)) {
118 return null;
119 }
120 final String target = getDetectText(text);
121 final LanguageResult result = detector.detect(target);
122 if (logger.isDebugEnabled()) {
123 logger.debug("detected lang:{}({}) from {}", result, result.getRawScore(), target);
124 }
125 return getSupportedLanguage(result.getLanguage());
126 }
127
128
129
130
131
132
133
134 protected String getDetectText(final String text) {
135 final String result;
136 if (text.length() <= maxTextLength) {
137 result = text;
138 } else {
139 result = text.substring(0, maxTextLength);
140 }
141 return result.replaceAll("\\s+", " ");
142 }
143
144
145
146
147
148
149
150 protected String getSupportedLanguage(final String lang) {
151 if (StringUtil.isBlank(lang)) {
152 return null;
153 }
154 for (final String l : supportedLanguages) {
155 if (l.equals(lang)) {
156 return l;
157 }
158 }
159 return null;
160 }
161
162
163
164
165
166
167 public void setDetector(final LanguageDetector detector) {
168 this.detector = detector;
169 }
170
171
172
173
174
175
176
177
178 public Script createScript(final Map<String, Object> doc, final String code) {
179 final StringBuilder buf = new StringBuilder(100);
180 buf.append(code);
181 final FessConfig fessConfig = ComponentUtil.getFessConfig();
182 final String language = DocumentUtil.getValue(doc, fessConfig.getIndexFieldLang(), String.class);
183 if (StringUtil.isNotBlank(language)) {
184 for (final String f : langFields) {
185 buf.append(";ctx._source.").append(f).append('_').append(language).append("=ctx._source.").append(f);
186 }
187 }
188 if (logger.isDebugEnabled()) {
189 logger.debug("update script: {}", buf);
190 }
191 return new Script(buf.toString());
192 }
193
194
195
196
197
198
199 public String getReindexScriptSource() {
200 final FessConfig fessConfig = ComponentUtil.getFessConfig();
201 final String langField = fessConfig.getIndexFieldLang();
202 final String code = Arrays.stream(langFields)
203 .map(s -> "ctx._source['" + s + "_'+ctx._source." + langField + "]=ctx._source." + s)
204 .collect(Collectors.joining(";"));
205 if (logger.isDebugEnabled()) {
206 logger.debug("reindex script: {}", code);
207 }
208 return "if(ctx._source." + langField + "!=null){" + code + "}";
209 }
210
211 }