1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.opensearch.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Map.Entry;
25 import java.util.function.Supplier;
26 import java.util.regex.Pattern;
27 import java.util.stream.Collectors;
28
29 import org.apache.http.auth.UsernamePasswordCredentials;
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.codelibs.core.lang.StringUtil;
33 import org.codelibs.fess.Constants;
34 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
35 import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
36 import org.codelibs.fess.crawler.client.ftp.FtpClient;
37 import org.codelibs.fess.crawler.client.http.HcHttpClient;
38 import org.codelibs.fess.crawler.client.http.config.CredentialsConfig;
39 import org.codelibs.fess.crawler.client.http.config.CredentialsConfig.CredentialsType;
40 import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig;
41 import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig.AuthSchemeType;
42 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
43 import org.codelibs.fess.crawler.client.smb.SmbClient;
44 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
45 import org.codelibs.fess.opensearch.config.bsentity.BsDataConfig;
46 import org.codelibs.fess.util.ParameterUtil;
47
48
49
50
51 public class DataConfig extends BsDataConfig implements CrawlingConfig {
52
53 private static final long serialVersionUID = 1L;
54
55 private static final Logger logger = LogManager.getLogger(DataConfig.class);
56
57 private static final String CRAWLER_WEB_PREFIX = "crawler.web.";
58
59 private static final String CRAWLER_WEB_HEADER_PREFIX = CRAWLER_WEB_PREFIX + "header.";
60
61 private static final String CRAWLER_WEB_AUTH = CRAWLER_WEB_PREFIX + "auth";
62
63 private static final String CRAWLER_USERAGENT = "crawler.useragent";
64
65 private static final String CRAWLER_PARAM_PREFIX = "crawler.param.";
66
67 private static final Object CRAWLER_FILE_AUTH = "crawler.file.auth";
68
69 protected Pattern[] includedDocPathPatterns;
70
71 protected Pattern[] excludedDocPathPatterns;
72
73 protected Map<String, String> handlerParameterMap;
74
75 protected Map<String, String> handlerScriptMap;
76
77 protected CrawlerClientFactory crawlerClientFactory = null;
78
79 protected Map<ConfigName, Map<String, String>> configParameterMap;
80
81 public DataConfig() {
82 setBoost(1.0f);
83 }
84
85 @Override
86 public String getDocumentBoost() {
87 return getBoost().toString();
88 }
89
90 public String getBoostValue() {
91 if (boost != null) {
92 return boost.toString();
93 }
94 return null;
95 }
96
97 public void setBoostValue(final String value) {
98 if (value != null) {
99 try {
100 boost = Float.parseFloat(value);
101 } catch (final Exception e) {
102 if (logger.isDebugEnabled()) {
103 logger.debug("Invalid boost value, keeping current: value={}, error={}", value, e.getMessage());
104 }
105 }
106 }
107 }
108
109 @Override
110 public String getIndexingTarget(final String input) {
111
112 return Constants.TRUE;
113 }
114
115 @Override
116 public String getConfigId() {
117 return ConfigType.DATA.getConfigId(getId());
118 }
119
120 public Map<String, String> getHandlerParameterMap() {
121 if (handlerParameterMap == null) {
122 handlerParameterMap = ParameterUtil.parse(getHandlerParameter());
123 }
124 return handlerParameterMap;
125 }
126
127 public Map<String, String> getHandlerScriptMap() {
128 if (handlerScriptMap == null) {
129 handlerScriptMap = ParameterUtil.parse(getHandlerScript());
130 }
131 return handlerScriptMap;
132 }
133
134 @Override
135 public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
136 if (crawlerClientFactory != null) {
137 return crawlerClientFactory;
138 }
139 final CrawlerClientFactory factory = creator.get();
140
141 final Map<String, String> paramMap = getHandlerParameterMap();
142
143 final Map<String, Object> factoryParamMap = new HashMap<>();
144 factory.setInitParameterMap(factoryParamMap);
145
146
147 for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
148 final String key = entry.getKey();
149 if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
150 factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
151 }
152 }
153
154
155 final String userAgent = paramMap.get(CRAWLER_USERAGENT);
156 if (StringUtil.isNotBlank(userAgent)) {
157 factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
158 }
159
160
161 final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
162 if (StringUtil.isNotBlank(webAuthStr)) {
163 final String[] webAuthNames = webAuthStr.split(",");
164 final List<WebAuthenticationConfig> authConfigList = new ArrayList<>();
165 for (final String webAuthName : webAuthNames) {
166 authConfigList.add(createWebAuthConfig(paramMap, webAuthName));
167 }
168 factoryParamMap.put(HcHttpClient.AUTHENTICATIONS_PROPERTY,
169 authConfigList.toArray(new WebAuthenticationConfig[authConfigList.size()]));
170 }
171
172
173 final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
174 int count = 1;
175 String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
176 while (StringUtil.isNotBlank(headerName)) {
177 final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
178 rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
179 count++;
180 headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
181 }
182 if (!rhList.isEmpty()) {
183 factoryParamMap.put(HcHttpClient.REQUEST_HEADERS_PROPERTY,
184 rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
185 }
186
187
188 final String proxyHost = paramMap.get(CRAWLER_WEB_PREFIX + "proxyHost");
189 final String proxyPort = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPort");
190 if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
191 factoryParamMap.put(HcHttpClient.PROXY_HOST_PROPERTY, proxyHost);
192 factoryParamMap.put(HcHttpClient.PROXY_PORT_PROPERTY, proxyPort);
193 final String proxyUsername = paramMap.get(CRAWLER_WEB_PREFIX + "proxyUsername");
194 final String proxyPassword = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPassword");
195 if (proxyUsername != null && proxyPassword != null) {
196 factoryParamMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
197 }
198 } else {
199 initializeDefaultHttpProxy(factoryParamMap);
200 }
201
202
203 final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
204 if (StringUtil.isNotBlank(fileAuthStr)) {
205 final String[] fileAuthNames = fileAuthStr.split(",");
206 final List<SmbAuthentication> smbAuthList = new ArrayList<>();
207 final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
208 final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
209 for (final String fileAuthName : fileAuthNames) {
210 final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
211 if (Constants.SAMBA.equals(scheme)) {
212 final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
213 final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
214 final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
215 final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
216 final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
217
218 if (StringUtil.isEmpty(username)) {
219 logger.warn("username is empty. fileAuth:{}", fileAuthName);
220 continue;
221 }
222
223 final SmbAuthentication smbAuth = new SmbAuthentication();
224 smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
225 smbAuth.setServer(hostname);
226 if (StringUtil.isNotBlank(port)) {
227 try {
228 smbAuth.setPort(Integer.parseInt(port));
229 } catch (final NumberFormatException e) {
230 logger.warn("Failed to parse {}", port, e);
231 }
232 }
233 smbAuth.setUsername(username);
234 smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
235 smbAuthList.add(smbAuth);
236
237 final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth =
238 new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
239 smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
240 smb1Auth.setServer(hostname);
241 if (StringUtil.isNotBlank(port)) {
242 try {
243 smb1Auth.setPort(Integer.parseInt(port));
244 } catch (final NumberFormatException e) {
245 logger.warn("Failed to parse {}", port, e);
246 }
247 }
248 smb1Auth.setUsername(username);
249 smb1Auth.setPassword(password == null ? StringUtil.EMPTY : password);
250 smb1AuthList.add(smb1Auth);
251 } else if (Constants.FTP.equals(scheme)) {
252 final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
253 final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
254 final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
255 final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
256
257 if (StringUtil.isEmpty(username)) {
258 logger.warn("username is empty. fileAuth:{}", fileAuthName);
259 continue;
260 }
261
262 final FtpAuthentication ftpAuth = new FtpAuthentication();
263 ftpAuth.setServer(hostname);
264 if (StringUtil.isNotBlank(port)) {
265 try {
266 ftpAuth.setPort(Integer.parseInt(port));
267 } catch (final NumberFormatException e) {
268 logger.warn("Failed to parse {}", port, e);
269 }
270 }
271 ftpAuth.setUsername(username);
272 ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
273 ftpAuthList.add(ftpAuth);
274 }
275 }
276 if (!smbAuthList.isEmpty()) {
277 factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
278 }
279 if (!smb1AuthList.isEmpty()) {
280 factoryParamMap.put(org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY,
281 smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
282 }
283 if (!ftpAuthList.isEmpty()) {
284 factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
285 }
286 }
287
288 crawlerClientFactory = factory;
289 return factory;
290 }
291
292 private WebAuthenticationConfig createWebAuthConfig(final Map<String, String> paramMap, final String webAuthName) {
293 final String prefix = CRAWLER_WEB_AUTH + "." + webAuthName + ".";
294 final String scheme = paramMap.get(prefix + "scheme");
295
296 final String username = paramMap.get(prefix + "username");
297 if (StringUtil.isEmpty(username)) {
298 throw new CrawlerSystemException("username is empty. webAuth:" + webAuthName);
299 }
300
301 final WebAuthenticationConfig config = new WebAuthenticationConfig();
302
303
304 final String host = paramMap.get(prefix + "host");
305 if (StringUtil.isNotBlank(host)) {
306 config.setHost(host);
307 }
308
309 final String port = paramMap.get(prefix + "port");
310 if (StringUtil.isNotBlank(port)) {
311 try {
312 config.setPort(Integer.parseInt(port));
313 } catch (final NumberFormatException e) {
314 logger.warn("Failed to parse port: {}", port, e);
315 }
316 }
317
318 final String realm = paramMap.get(prefix + "realm");
319 if (StringUtil.isNotBlank(realm)) {
320 config.setRealm(realm);
321 }
322
323
324 if (Constants.BASIC.equals(scheme)) {
325 config.setAuthSchemeType(AuthSchemeType.BASIC);
326 } else if (Constants.DIGEST.equals(scheme)) {
327 config.setAuthSchemeType(AuthSchemeType.DIGEST);
328 } else if (Constants.NTLM.equals(scheme)) {
329 config.setAuthSchemeType(AuthSchemeType.NTLM);
330
331 final Map<String, String> jcifsParams = paramMap.entrySet()
332 .stream()
333 .filter(e -> e.getKey().startsWith("jcifs."))
334 .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
335 if (!jcifsParams.isEmpty()) {
336 config.setNtlmParameters(jcifsParams);
337 }
338 } else if (Constants.FORM.equals(scheme)) {
339 config.setAuthSchemeType(AuthSchemeType.FORM);
340 final Map<String, String> formParams = paramMap.entrySet()
341 .stream()
342 .filter(e -> e.getKey().startsWith(prefix))
343 .collect(Collectors.toMap(e -> e.getKey().substring(prefix.length()), Entry::getValue));
344 config.setFormParameters(formParams);
345 }
346
347
348 final CredentialsConfig credentials = new CredentialsConfig();
349 credentials.setUsername(username);
350 final String password = paramMap.get(prefix + "password");
351 credentials.setPassword(password == null ? StringUtil.EMPTY : password);
352 if (Constants.NTLM.equals(scheme)) {
353 credentials.setType(CredentialsType.NTLM);
354 credentials.setDomain(paramMap.get(prefix + "domain"));
355 credentials.setWorkstation(paramMap.get(prefix + "workstation"));
356 }
357 config.setCredentials(credentials);
358
359 return config;
360 }
361
362 @Override
363 public Map<String, String> getConfigParameterMap(final ConfigName name) {
364 if (configParameterMap == null) {
365 configParameterMap = ParameterUtil.createConfigParameterMap(getHandlerParameter());
366 }
367
368 final Map<String, String> configMap = configParameterMap.get(name);
369 if (configMap == null) {
370 return Collections.emptyMap();
371 }
372 return configMap;
373 }
374
375 @Override
376 public String getId() {
377 return asDocMeta().id();
378 }
379
380 public void setId(final String id) {
381 asDocMeta().id(id);
382 }
383
384 public Long getVersionNo() {
385 return asDocMeta().version();
386 }
387
388 public void setVersionNo(final Long version) {
389 asDocMeta().version(version);
390 }
391
392 @Override
393 public Integer getTimeToLive() {
394 String value = getHandlerParameterMap().get("time_to_live");
395 if (StringUtil.isBlank(value)) {
396 value = getHandlerParameterMap().get("timeToLive");
397 if (StringUtil.isBlank(value)) {
398 return null;
399 }
400 logger.warn("timeToLive is deprecated. Please use time_to_live.");
401 }
402 try {
403 return Integer.parseInt(value);
404 } catch (final NumberFormatException e) {
405 if (logger.isDebugEnabled()) {
406 logger.debug("Invalid format: {}", value, e);
407 }
408 }
409 return null;
410 }
411
412 @Override
413 public String toString() {
414 return "DataConfig [available=" + available + ", boost=" + boost + ", createdBy=" + createdBy + ", createdTime=" + createdTime
415 + ", handlerName=" + handlerName + ", handlerParameter=" + handlerParameter + ", handlerScript=" + handlerScript + ", name="
416 + name + ", permissions=" + Arrays.toString(permissions) + ", sortOrder=" + sortOrder + ", updatedBy=" + updatedBy
417 + ", updatedTime=" + updatedTime + "]";
418 }
419 }