1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.es.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Map.Entry;
25 import java.util.Properties;
26 import java.util.function.Supplier;
27 import java.util.regex.Pattern;
28 import java.util.stream.Collectors;
29
30 import org.apache.http.auth.AuthScheme;
31 import org.apache.http.auth.AuthScope;
32 import org.apache.http.auth.Credentials;
33 import org.apache.http.auth.NTCredentials;
34 import org.apache.http.auth.UsernamePasswordCredentials;
35 import org.apache.http.impl.auth.BasicScheme;
36 import org.apache.http.impl.auth.DigestScheme;
37 import org.apache.http.impl.auth.NTLMScheme;
38 import org.apache.logging.log4j.LogManager;
39 import org.apache.logging.log4j.Logger;
40 import org.codelibs.core.lang.StringUtil;
41 import org.codelibs.fess.Constants;
42 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
43 import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
44 import org.codelibs.fess.crawler.client.ftp.FtpClient;
45 import org.codelibs.fess.crawler.client.http.Authentication;
46 import org.codelibs.fess.crawler.client.http.HcHttpClient;
47 import org.codelibs.fess.crawler.client.http.form.FormScheme;
48 import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;
49 import org.codelibs.fess.crawler.client.http.ntlm.JcifsEngine;
50 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
51 import org.codelibs.fess.crawler.client.smb.SmbClient;
52 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
53 import org.codelibs.fess.es.config.bsentity.BsDataConfig;
54 import org.codelibs.fess.util.ParameterUtil;
55
56
57
58
59 public class DataConfig extends BsDataConfig implements CrawlingConfig {
60
61 private static final long serialVersionUID = 1L;
62
63 private static final Logger logger = LogManager.getLogger(DataConfig.class);
64
65 private static final String CRAWLER_WEB_PREFIX = "crawler.web.";
66
67 private static final String CRAWLER_WEB_HEADER_PREFIX = CRAWLER_WEB_PREFIX + "header.";
68
69 private static final String CRAWLER_WEB_AUTH = CRAWLER_WEB_PREFIX + "auth";
70
71 private static final String CRAWLER_USERAGENT = "crawler.useragent";
72
73 private static final String CRAWLER_PARAM_PREFIX = "crawler.param.";
74
75 private static final Object CRAWLER_FILE_AUTH = "crawler.file.auth";
76
77 protected Pattern[] includedDocPathPatterns;
78
79 protected Pattern[] excludedDocPathPatterns;
80
81 protected Map<String, String> handlerParameterMap;
82
83 protected Map<String, String> handlerScriptMap;
84
85 protected CrawlerClientFactory crawlerClientFactory = null;
86
87 public DataConfig() {
88 setBoost(1.0f);
89 }
90
91 @Override
92 public String getDocumentBoost() {
93 return getBoost().toString();
94 }
95
96 public String getBoostValue() {
97 if (boost != null) {
98 return boost.toString();
99 }
100 return null;
101 }
102
103 public void setBoostValue(final String value) {
104 if (value != null) {
105 try {
106 boost = Float.parseFloat(value);
107 } catch (final Exception e) {}
108 }
109 }
110
111 @Override
112 public String getIndexingTarget(final String input) {
113
114 return Constants.TRUE;
115 }
116
117 @Override
118 public String getConfigId() {
119 return ConfigType.DATA.getConfigId(getId());
120 }
121
122 public Map<String, String> getHandlerParameterMap() {
123 if (handlerParameterMap == null) {
124 handlerParameterMap = ParameterUtil.parse(getHandlerParameter());
125 }
126 return handlerParameterMap;
127 }
128
129 public Map<String, String> getHandlerScriptMap() {
130 if (handlerScriptMap == null) {
131 handlerScriptMap = ParameterUtil.parse(getHandlerScript());
132 }
133 return handlerScriptMap;
134 }
135
136 @Override
137 public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
138 if (crawlerClientFactory != null) {
139 return crawlerClientFactory;
140 }
141 final CrawlerClientFactory factory = creator.get();
142
143 final Map<String, String> paramMap = getHandlerParameterMap();
144
145 final Map<String, Object> factoryParamMap = new HashMap<>();
146 factory.setInitParameterMap(factoryParamMap);
147
148
149 for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
150 final String key = entry.getKey();
151 if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
152 factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
153 }
154 }
155
156
157 final String userAgent = paramMap.get(CRAWLER_USERAGENT);
158 if (StringUtil.isNotBlank(userAgent)) {
159 factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
160 }
161
162
163 final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
164 if (StringUtil.isNotBlank(webAuthStr)) {
165 final String[] webAuthNames = webAuthStr.split(",");
166 final List<Authentication> basicAuthList = new ArrayList<>();
167 for (final String webAuthName : webAuthNames) {
168 final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
169
170 final AuthScheme authScheme = getAuthScheme(paramMap, webAuthName, scheme);
171 final AuthScope authScope = getAuthScope(webAuthName, scheme, paramMap);
172 final Credentials credentials = getCredentials(webAuthName, scheme, paramMap);
173 basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
174 }
175 factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY,
176 basicAuthList.toArray(new Authentication[basicAuthList.size()]));
177 }
178
179
180 final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
181 int count = 1;
182 String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
183 while (StringUtil.isNotBlank(headerName)) {
184 final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
185 rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
186 count++;
187 headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
188 }
189 if (!rhList.isEmpty()) {
190 factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
191 rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
192 }
193
194
195 final String proxyHost = paramMap.get(CRAWLER_WEB_PREFIX + "proxyHost");
196 final String proxyPort = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPort");
197 if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
198 factoryParamMap.put(HcHttpClient.PROXY_HOST_PROPERTY, proxyHost);
199 factoryParamMap.put(HcHttpClient.PROXY_PORT_PROPERTY, proxyPort);
200 final String proxyUsername = paramMap.get(CRAWLER_WEB_PREFIX + "proxyUsername");
201 final String proxyPassword = paramMap.get(CRAWLER_WEB_PREFIX + "proxyPassword");
202 if (proxyUsername != null && proxyPassword != null) {
203 factoryParamMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
204 }
205 } else {
206 initializeDefaultHttpProxy(factoryParamMap);
207 }
208
209
210 final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
211 if (StringUtil.isNotBlank(fileAuthStr)) {
212 final String[] fileAuthNames = fileAuthStr.split(",");
213 final List<SmbAuthentication> smbAuthList = new ArrayList<>();
214 final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
215 final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
216 for (final String fileAuthName : fileAuthNames) {
217 final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
218 if (Constants.SAMBA.equals(scheme)) {
219 final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
220 final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
221 final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
222 final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
223 final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
224
225 if (StringUtil.isEmpty(username)) {
226 logger.warn("username is empty. fileAuth:{}", fileAuthName);
227 continue;
228 }
229
230 final SmbAuthentication smbAuth = new SmbAuthentication();
231 smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
232 smbAuth.setServer(hostname);
233 if (StringUtil.isNotBlank(port)) {
234 try {
235 smbAuth.setPort(Integer.parseInt(port));
236 } catch (final NumberFormatException e) {
237 logger.warn("Failed to parse {}", port, e);
238 }
239 }
240 smbAuth.setUsername(username);
241 smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
242 smbAuthList.add(smbAuth);
243
244 final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth =
245 new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
246 smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
247 smb1Auth.setServer(hostname);
248 if (StringUtil.isNotBlank(port)) {
249 try {
250 smb1Auth.setPort(Integer.parseInt(port));
251 } catch (final NumberFormatException e) {
252 logger.warn("Failed to parse {}", port, e);
253 }
254 }
255 smb1Auth.setUsername(username);
256 smb1Auth.setPassword(password == null ? StringUtil.EMPTY : password);
257 smb1AuthList.add(smb1Auth);
258 } else if (Constants.FTP.equals(scheme)) {
259 final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
260 final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
261 final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
262 final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
263
264 if (StringUtil.isEmpty(username)) {
265 logger.warn("username is empty. fileAuth:{}", fileAuthName);
266 continue;
267 }
268
269 final FtpAuthentication ftpAuth = new FtpAuthentication();
270 ftpAuth.setServer(hostname);
271 if (StringUtil.isNotBlank(port)) {
272 try {
273 ftpAuth.setPort(Integer.parseInt(port));
274 } catch (final NumberFormatException e) {
275 logger.warn("Failed to parse {}", port, e);
276 }
277 }
278 ftpAuth.setUsername(username);
279 ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
280 ftpAuthList.add(ftpAuth);
281 }
282 }
283 if (!smbAuthList.isEmpty()) {
284 factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
285 }
286 if (!smb1AuthList.isEmpty()) {
287 factoryParamMap.put(org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY,
288 smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
289 }
290 if (!ftpAuthList.isEmpty()) {
291 factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
292 }
293 }
294
295 crawlerClientFactory = factory;
296 return factory;
297 }
298
299 private AuthScheme getAuthScheme(final Map<String, String> paramMap, final String webAuthName, final String scheme) {
300 AuthScheme authScheme = null;
301 if (Constants.BASIC.equals(scheme)) {
302 authScheme = new BasicScheme();
303 } else if (Constants.DIGEST.equals(scheme)) {
304 authScheme = new DigestScheme();
305 } else if (Constants.NTLM.equals(scheme)) {
306 final Properties props = new Properties();
307 paramMap.entrySet().stream().filter(e -> e.getKey().startsWith("jcifs.")).forEach(e -> {
308 props.setProperty(e.getKey(), e.getValue());
309 });
310 authScheme = new NTLMScheme(new JcifsEngine(props));
311 } else if (Constants.FORM.equals(scheme)) {
312 final String prefix = CRAWLER_WEB_AUTH + "." + webAuthName + ".";
313 final Map<String, String> parameterMap = paramMap.entrySet().stream().filter(e -> e.getKey().startsWith(prefix))
314 .collect(Collectors.toMap(e -> e.getKey().substring(prefix.length()), Entry::getValue));
315 authScheme = new FormScheme(parameterMap);
316 }
317 return authScheme;
318 }
319
320 private Credentials getCredentials(final String webAuthName, final String scheme, final Map<String, String> paramMap) {
321 final String username = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".username");
322 if (StringUtil.isEmpty(username)) {
323 throw new CrawlerSystemException("username is empty. webAuth:" + webAuthName);
324 }
325 final String password = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".password");
326 Credentials credentials;
327 if (Constants.NTLM.equals(scheme)) {
328 final String workstation = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".workstation");
329 final String domain = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".domain");
330 credentials = new NTCredentials(username, password == null ? StringUtil.EMPTY : password,
331 workstation == null ? StringUtil.EMPTY : workstation, domain == null ? StringUtil.EMPTY : domain);
332 } else {
333 credentials = new UsernamePasswordCredentials(username, password == null ? StringUtil.EMPTY : password);
334 }
335 return credentials;
336 }
337
338 private AuthScope getAuthScope(final String webAuthName, final String scheme, final Map<String, String> paramMap) {
339 final String hostname = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".host");
340 final String port = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".port");
341 final String realm = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".realm");
342 AuthScope authScope;
343 if (StringUtil.isBlank(hostname)) {
344 authScope = AuthScope.ANY;
345 } else {
346 int p = AuthScope.ANY_PORT;
347 if (StringUtil.isNotBlank(port)) {
348 try {
349 p = Integer.parseInt(port);
350 } catch (final NumberFormatException e) {
351 logger.warn("Failed to parse {}", port, e);
352 }
353 }
354
355 String r = realm;
356 if (StringUtil.isBlank(realm)) {
357 r = AuthScope.ANY_REALM;
358 }
359
360 String s = scheme;
361 if (StringUtil.isBlank(scheme) || Constants.NTLM.equals(scheme)) {
362 s = AuthScope.ANY_SCHEME;
363 }
364 authScope = new AuthScope(hostname, p, r, s);
365 }
366 return authScope;
367 }
368
369 @Override
370 public Map<String, String> getConfigParameterMap(final ConfigName name) {
371 return Collections.emptyMap();
372 }
373
374 @Override
375 public String getId() {
376 return asDocMeta().id();
377 }
378
379 public void setId(final String id) {
380 asDocMeta().id(id);
381 }
382
383 public Long getVersionNo() {
384 return asDocMeta().version();
385 }
386
387 public void setVersionNo(final Long version) {
388 asDocMeta().version(version);
389 }
390
391 @Override
392 public Integer getTimeToLive() {
393 final String value = getHandlerParameterMap().get("timeToLive");
394 if (StringUtil.isBlank(value)) {
395 return null;
396 }
397 try {
398 return Integer.parseInt(value);
399 } catch (final NumberFormatException e) {
400 if (logger.isDebugEnabled()) {
401 logger.debug("Invalid format: {}", value, e);
402 }
403 }
404 return null;
405 }
406
407 @Override
408 public String toString() {
409 return "DataConfig [available=" + available + ", boost=" + boost + ", createdBy=" + createdBy + ", createdTime=" + createdTime
410 + ", handlerName=" + handlerName + ", handlerParameter=" + handlerParameter + ", handlerScript=" + handlerScript + ", name="
411 + name + ", permissions=" + Arrays.toString(permissions) + ", sortOrder=" + sortOrder + ", updatedBy=" + updatedBy
412 + ", updatedTime=" + updatedTime + "]";
413 }
414 }