1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.es.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.regex.Pattern;
25
26 import org.apache.http.auth.AuthScheme;
27 import org.apache.http.auth.AuthScope;
28 import org.apache.http.auth.Credentials;
29 import org.apache.http.auth.NTCredentials;
30 import org.apache.http.auth.UsernamePasswordCredentials;
31 import org.apache.http.impl.auth.BasicScheme;
32 import org.apache.http.impl.auth.DigestScheme;
33 import org.apache.http.impl.auth.NTLMScheme;
34 import org.codelibs.core.lang.StringUtil;
35 import org.codelibs.fess.Constants;
36 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
37 import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
38 import org.codelibs.fess.crawler.client.ftp.FtpClient;
39 import org.codelibs.fess.crawler.client.http.Authentication;
40 import org.codelibs.fess.crawler.client.http.HcHttpClient;
41 import org.codelibs.fess.crawler.client.http.impl.AuthenticationImpl;
42 import org.codelibs.fess.crawler.client.http.ntlm.JcifsEngine;
43 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
44 import org.codelibs.fess.crawler.client.smb.SmbClient;
45 import org.codelibs.fess.es.config.bsentity.BsDataConfig;
46 import org.codelibs.fess.es.config.exbhv.DataConfigToLabelBhv;
47 import org.codelibs.fess.es.config.exbhv.LabelTypeBhv;
48 import org.codelibs.fess.mylasta.direction.FessConfig;
49 import org.codelibs.fess.util.ComponentUtil;
50 import org.codelibs.fess.util.ParameterUtil;
51 import org.dbflute.cbean.result.ListResultBean;
52 import org.slf4j.Logger;
53 import org.slf4j.LoggerFactory;
54
55
56
57
58 public class DataConfig extends BsDataConfig implements CrawlingConfig {
59
60 private static final long serialVersionUID = 1L;
61
62 private static final Logger logger = LoggerFactory.getLogger(DataConfig.class);
63
64 private static final String CRAWLER_WEB_HEADER_PREFIX = "crawler.web.header.";
65
66 private static final String CRAWLER_WEB_AUTH = "crawler.web.auth";
67
68 private static final String CRAWLER_USERAGENT = "crawler.useragent";
69
70 private static final String CRAWLER_PARAM_PREFIX = "crawler.param.";
71
72 private static final Object CRAWLER_FILE_AUTH = "crawler.file.auth";
73
74 private String[] labelTypeIds;
75
76 protected Pattern[] includedDocPathPatterns;
77
78 protected Pattern[] excludedDocPathPatterns;
79
80 private Map<String, String> handlerParameterMap;
81
82 private Map<String, String> handlerScriptMap;
83
84 private volatile List<LabelType> labelTypeList;
85
86 public DataConfig() {
87 super();
88 setBoost(1.0f);
89 }
90
91 public String[] getLabelTypeIds() {
92 if (labelTypeIds == null) {
93 return StringUtil.EMPTY_STRINGS;
94 }
95 return labelTypeIds;
96 }
97
98 public void setLabelTypeIds(final String[] labelTypeIds) {
99 this.labelTypeIds = labelTypeIds;
100 }
101
102 public List<LabelType> getLabelTypeList() {
103 if (labelTypeList == null) {
104 synchronized (this) {
105 if (labelTypeList == null) {
106 final FessConfig fessConfig = ComponentUtil.getFessConfig();
107 final DataConfigToLabelBhv dataConfigToLabelBhv = ComponentUtil.getComponent(DataConfigToLabelBhv.class);
108 final ListResultBean<DataConfigToLabel> mappingList = dataConfigToLabelBhv.selectList(cb -> {
109 cb.query().setDataConfigId_Equal(getId());
110 cb.specify().columnLabelTypeId();
111 cb.paging(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger().intValue(), 1);
112 });
113 final List<String> labelIdList = new ArrayList<>();
114 for (final DataConfigToLabel mapping : mappingList) {
115 labelIdList.add(mapping.getLabelTypeId());
116 }
117 final LabelTypeBhv labelTypeBhv = ComponentUtil.getComponent(LabelTypeBhv.class);
118 labelTypeList = labelIdList.isEmpty() ? Collections.emptyList() : labelTypeBhv.selectList(cb -> {
119 cb.query().setId_InScope(labelIdList);
120 cb.query().addOrderBy_SortOrder_Asc();
121 cb.fetchFirst(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger());
122 });
123 }
124 }
125 }
126 return labelTypeList;
127 }
128
129 @Override
130 public String[] getLabelTypeValues() {
131 final List<LabelType> list = getLabelTypeList();
132 final List<String> labelValueList = new ArrayList<>(list.size());
133 for (final LabelType labelType : list) {
134 labelValueList.add(labelType.getValue());
135 }
136 return labelValueList.toArray(new String[labelValueList.size()]);
137 }
138
139 @Override
140 public String getDocumentBoost() {
141 return Float.valueOf(getBoost().floatValue()).toString();
142 }
143
144 public String getBoostValue() {
145 if (boost != null) {
146 return boost.toString();
147 }
148 return null;
149 }
150
151 public void setBoostValue(final String value) {
152 if (value != null) {
153 try {
154 boost = Float.parseFloat(value);
155 } catch (final Exception e) {}
156 }
157 }
158
159 @Override
160 public String getIndexingTarget(final String input) {
161
162 return Constants.TRUE;
163 }
164
165 @Override
166 public String getConfigId() {
167 return ConfigType.DATA.getConfigId(getId());
168 }
169
170 public Map<String, String> getHandlerParameterMap() {
171 if (handlerParameterMap == null) {
172 handlerParameterMap = ParameterUtil.parse(getHandlerParameter());
173 }
174 return handlerParameterMap;
175 }
176
177 public Map<String, String> getHandlerScriptMap() {
178 if (handlerScriptMap == null) {
179 handlerScriptMap = ParameterUtil.parse(getHandlerScript());
180 }
181 return handlerScriptMap;
182 }
183
184 @Override
185 public Map<String, Object> initializeClientFactory(final CrawlerClientFactory crawlerClientFactory) {
186 final Map<String, String> paramMap = getHandlerParameterMap();
187
188 final Map<String, Object> factoryParamMap = new HashMap<>();
189 crawlerClientFactory.setInitParameterMap(factoryParamMap);
190
191
192 for (final Map.Entry<String, String> entry : paramMap.entrySet()) {
193 final String key = entry.getKey();
194 if (key.startsWith(CRAWLER_PARAM_PREFIX)) {
195 factoryParamMap.put(key.substring(CRAWLER_PARAM_PREFIX.length()), entry.getValue());
196 }
197 }
198
199
200 final String userAgent = paramMap.get(CRAWLER_USERAGENT);
201 if (StringUtil.isNotBlank(userAgent)) {
202 factoryParamMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
203 }
204
205
206 final String webAuthStr = paramMap.get(CRAWLER_WEB_AUTH);
207 if (StringUtil.isNotBlank(webAuthStr)) {
208 final String[] webAuthNames = webAuthStr.split(",");
209 final List<Authentication> basicAuthList = new ArrayList<>();
210 for (final String webAuthName : webAuthNames) {
211 final String scheme = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".scheme");
212 final String hostname = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".host");
213 final String port = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".port");
214 final String realm = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".realm");
215 final String username = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".username");
216 final String password = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".password");
217
218 if (StringUtil.isEmpty(username)) {
219 logger.warn("username is empty. webAuth:" + webAuthName);
220 continue;
221 }
222
223 AuthScheme authScheme = null;
224 if (Constants.BASIC.equals(scheme)) {
225 authScheme = new BasicScheme();
226 } else if (Constants.DIGEST.equals(scheme)) {
227 authScheme = new DigestScheme();
228 } else if (Constants.NTLM.equals(scheme)) {
229 authScheme = new NTLMScheme(new JcifsEngine());
230 }
231
232
233 AuthScope authScope;
234 if (StringUtil.isBlank(hostname)) {
235 authScope = AuthScope.ANY;
236 } else {
237 int p = AuthScope.ANY_PORT;
238 if (StringUtil.isNotBlank(port)) {
239 try {
240 p = Integer.parseInt(port);
241 } catch (final NumberFormatException e) {
242 logger.warn("Failed to parse " + port, e);
243 }
244 }
245
246 String r = realm;
247 if (StringUtil.isBlank(realm)) {
248 r = AuthScope.ANY_REALM;
249 }
250
251 String s = scheme;
252 if (StringUtil.isBlank(scheme) || Constants.NTLM.equals(scheme)) {
253 s = AuthScope.ANY_SCHEME;
254 }
255 authScope = new AuthScope(hostname, p, r, s);
256 }
257
258 Credentials credentials;
259 if (Constants.NTLM.equals(scheme)) {
260 final String workstation = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".workstation");
261 final String domain = paramMap.get(CRAWLER_WEB_AUTH + "." + webAuthName + ".domain");
262 credentials =
263 new NTCredentials(username, password == null ? StringUtil.EMPTY : password,
264 workstation == null ? StringUtil.EMPTY : workstation, domain == null ? StringUtil.EMPTY : domain);
265 } else {
266 credentials = new UsernamePasswordCredentials(username, password == null ? StringUtil.EMPTY : password);
267 }
268
269 basicAuthList.add(new AuthenticationImpl(authScope, credentials, authScheme));
270 }
271 factoryParamMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY,
272 basicAuthList.toArray(new Authentication[basicAuthList.size()]));
273 }
274
275
276 final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
277 int count = 1;
278 String headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
279 while (StringUtil.isNotBlank(headerName)) {
280 final String headerValue = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".value");
281 rhList.add(new org.codelibs.fess.crawler.client.http.RequestHeader(headerName, headerValue));
282 count++;
283 headerName = paramMap.get(CRAWLER_WEB_HEADER_PREFIX + count + ".name");
284 }
285 if (!rhList.isEmpty()) {
286 factoryParamMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
287 rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
288 }
289
290
291 final String fileAuthStr = paramMap.get(CRAWLER_FILE_AUTH);
292 if (StringUtil.isNotBlank(fileAuthStr)) {
293 final String[] fileAuthNames = fileAuthStr.split(",");
294 final List<SmbAuthentication> smbAuthList = new ArrayList<>();
295 final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
296 for (final String fileAuthName : fileAuthNames) {
297 final String scheme = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".scheme");
298 if (Constants.SAMBA.equals(scheme)) {
299 final String domain = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".domain");
300 final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
301 final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
302 final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
303 final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
304
305 if (StringUtil.isEmpty(username)) {
306 logger.warn("username is empty. fileAuth:" + fileAuthName);
307 continue;
308 }
309
310 final SmbAuthentication smbAuth = new SmbAuthentication();
311 smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
312 smbAuth.setServer(hostname);
313 if (StringUtil.isNotBlank(port)) {
314 try {
315 smbAuth.setPort(Integer.parseInt(port));
316 } catch (final NumberFormatException e) {
317 logger.warn("Failed to parse " + port, e);
318 }
319 }
320 smbAuth.setUsername(username);
321 smbAuth.setPassword(password == null ? StringUtil.EMPTY : password);
322 smbAuthList.add(smbAuth);
323 } else if (Constants.FTP.equals(scheme)) {
324 final String hostname = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".host");
325 final String port = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".port");
326 final String username = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".username");
327 final String password = paramMap.get(CRAWLER_FILE_AUTH + "." + fileAuthName + ".password");
328
329 if (StringUtil.isEmpty(username)) {
330 logger.warn("username is empty. fileAuth:" + fileAuthName);
331 continue;
332 }
333
334 final FtpAuthentication ftpAuth = new FtpAuthentication();
335 ftpAuth.setServer(hostname);
336 if (StringUtil.isNotBlank(port)) {
337 try {
338 ftpAuth.setPort(Integer.parseInt(port));
339 } catch (final NumberFormatException e) {
340 logger.warn("Failed to parse " + port, e);
341 }
342 }
343 ftpAuth.setUsername(username);
344 ftpAuth.setPassword(password == null ? StringUtil.EMPTY : password);
345 ftpAuthList.add(ftpAuth);
346 }
347 }
348 if (!smbAuthList.isEmpty()) {
349 factoryParamMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
350 }
351 if (!ftpAuthList.isEmpty()) {
352 factoryParamMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
353 }
354 }
355
356 return factoryParamMap;
357 }
358
359 @Override
360 public Map<String, String> getConfigParameterMap(final ConfigName name) {
361 return Collections.emptyMap();
362 }
363
364 @Override
365 public String getId() {
366 return asDocMeta().id();
367 }
368
369 public void setId(final String id) {
370 asDocMeta().id(id);
371 }
372
373 public Long getVersionNo() {
374 return asDocMeta().version();
375 }
376
377 public void setVersionNo(final Long version) {
378 asDocMeta().version(version);
379 }
380
381 @Override
382 public Integer getTimeToLive() {
383 final String value = getHandlerParameterMap().get("timeToLive");
384 if (StringUtil.isBlank(value)) {
385 return null;
386 }
387 try {
388 return Integer.parseInt(value);
389 } catch (final NumberFormatException e) {
390 if (logger.isDebugEnabled()) {
391 logger.debug("Invalid format: " + value, e);
392 }
393 }
394 return null;
395 }
396
397 @Override
398 public String toString() {
399 return "DataConfig [available=" + available + ", boost=" + boost + ", createdBy=" + createdBy + ", createdTime=" + createdTime
400 + ", handlerName=" + handlerName + ", handlerParameter=" + handlerParameter + ", handlerScript=" + handlerScript
401 + ", name=" + name + ", permissions=" + Arrays.toString(permissions) + ", sortOrder=" + sortOrder + ", updatedBy="
402 + updatedBy + ", updatedTime=" + updatedTime + "]";
403 }
404
405 }