1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.es.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.function.Supplier;
25 import java.util.regex.Pattern;
26
27 import org.apache.http.auth.UsernamePasswordCredentials;
28 import org.codelibs.core.lang.StringUtil;
29 import org.codelibs.fess.Constants;
30 import org.codelibs.fess.app.service.RequestHeaderService;
31 import org.codelibs.fess.app.service.WebAuthenticationService;
32 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
33 import org.codelibs.fess.crawler.client.http.Authentication;
34 import org.codelibs.fess.crawler.client.http.HcHttpClient;
35 import org.codelibs.fess.es.config.bsentity.BsWebConfig;
36 import org.codelibs.fess.es.config.exentity.CrawlingConfig.Param.Client;
37 import org.codelibs.fess.helper.SystemHelper;
38 import org.codelibs.fess.mylasta.direction.FessConfig;
39 import org.codelibs.fess.util.ComponentUtil;
40 import org.codelibs.fess.util.ParameterUtil;
41
42
43
44
45 public class WebConfig extends BsWebConfig implements CrawlingConfig {
46
47 private static final long serialVersionUID = 1L;
48
49 protected volatile Pattern[] includedDocUrlPatterns;
50
51 protected volatile Pattern[] excludedDocUrlPatterns;
52
53 protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
54
55 protected CrawlerClientFactory crawlerClientFactory = null;
56
57 public WebConfig() {
58 setBoost(1.0f);
59 }
60
61 @Override
62 public String getDocumentBoost() {
63 return getBoost().toString();
64 }
65
66 @Override
67 public String getIndexingTarget(final String input) {
68 if (includedDocUrlPatterns == null || excludedDocUrlPatterns == null) {
69 initDocUrlPattern();
70 }
71
72 if (includedDocUrlPatterns.length == 0 && excludedDocUrlPatterns.length == 0) {
73 return Constants.TRUE;
74 }
75
76 for (final Pattern pattern : includedDocUrlPatterns) {
77 if (pattern.matcher(input).matches()) {
78 return Constants.TRUE;
79 }
80 }
81
82 for (final Pattern pattern : excludedDocUrlPatterns) {
83 if (pattern.matcher(input).matches()) {
84 return Constants.FALSE;
85 }
86 }
87
88 return Constants.TRUE;
89 }
90
91 protected synchronized void initDocUrlPattern() {
92
93 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
94 if (includedDocUrlPatterns == null) {
95 if (StringUtil.isNotBlank(getIncludedDocUrls())) {
96 final List<Pattern> urlPatterList = new ArrayList<>();
97 final String[] urls = getIncludedDocUrls().split("[\r\n]");
98 for (final String u : urls) {
99 final String v = systemHelper.normalizeConfigPath(u);
100 if (StringUtil.isNotBlank(v)) {
101 urlPatterList.add(Pattern.compile(v));
102 }
103 }
104 includedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
105 } else {
106 includedDocUrlPatterns = new Pattern[0];
107 }
108 }
109
110 if (excludedDocUrlPatterns == null) {
111 if (StringUtil.isNotBlank(getExcludedDocUrls())) {
112 final List<Pattern> urlPatterList = new ArrayList<>();
113 final String[] urls = getExcludedDocUrls().split("[\r\n]");
114 for (final String u : urls) {
115 final String v = systemHelper.normalizeConfigPath(u);
116 if (StringUtil.isNotBlank(v)) {
117 urlPatterList.add(Pattern.compile(v));
118 }
119 }
120 excludedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
121 } else if (includedDocUrlPatterns.length > 0) {
122 excludedDocUrlPatterns = new Pattern[] { Pattern.compile(".*") };
123 } else {
124 excludedDocUrlPatterns = new Pattern[0];
125 }
126 }
127 }
128
129 public String getBoostValue() {
130 if (boost != null) {
131 return boost.toString();
132 }
133 return null;
134 }
135
136 public void setBoostValue(final String value) {
137 if (value != null) {
138 try {
139 boost = Float.parseFloat(value);
140 } catch (final Exception e) {}
141 }
142 }
143
144 @Override
145 public String getConfigId() {
146 return ConfigType.WEB.getConfigId(getId());
147 }
148
149 @Override
150 public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
151 if (crawlerClientFactory != null) {
152 return crawlerClientFactory;
153 }
154 final CrawlerClientFactory factory = creator.get();
155
156 final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
157 final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
158 final FessConfig fessConfig = ComponentUtil.getFessConfig();
159
160
161 final Map<String, Object> paramMap = new HashMap<>();
162 factory.setInitParameterMap(paramMap);
163
164 final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
165 if (clientConfigMap != null) {
166 paramMap.putAll(clientConfigMap);
167 }
168
169
170 if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
171 paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
172 }
173
174 final String userAgent = getUserAgent();
175 if (StringUtil.isNotBlank(userAgent)) {
176 paramMap.put(Client.USER_AGENT, userAgent);
177 }
178
179 final List<WebAuthentication> webAuthList = webAuthenticationService.getWebAuthenticationList(getId());
180 final List<Authentication> basicAuthList = new ArrayList<>();
181 for (final WebAuthentication webAuth : webAuthList) {
182 basicAuthList.add(webAuth.getAuthentication());
183 }
184 paramMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
185
186
187 final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
188 final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
189 for (final RequestHeader requestHeader : requestHeaderList) {
190 rhList.add(requestHeader.getCrawlerRequestHeader());
191 }
192 paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
193 rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
194
195 final String proxyHost = (String) paramMap.get(Param.Client.PROXY_HOST);
196 final String proxyPort = (String) paramMap.get(Param.Client.PROXY_PORT);
197 if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
198
199 if (paramMap.get(Param.Client.PROXY_USERNAME) != null && paramMap.get(Param.Client.PROXY_PASSWORD) != null) {
200 paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(
201 paramMap.remove(Param.Client.PROXY_USERNAME).toString(), paramMap.remove(Param.Client.PROXY_PASSWORD).toString()));
202 }
203 } else {
204 initializeDefaultHttpProxy(paramMap);
205 }
206
207 crawlerClientFactory = factory;
208 return factory;
209 }
210
211 @Override
212 public Map<String, String> getConfigParameterMap(final ConfigName name) {
213 if (configParameterMap == null) {
214 configParameterMap = ParameterUtil.createConfigParameterMap(getConfigParameter());
215 }
216
217 final Map<String, String> configMap = configParameterMap.get(name);
218 if (configMap == null) {
219 return Collections.emptyMap();
220 }
221 return configMap;
222 }
223
224 @Override
225 public String getId() {
226 return asDocMeta().id();
227 }
228
229 public void setId(final String id) {
230 asDocMeta().id(id);
231 }
232
233 public Long getVersionNo() {
234 return asDocMeta().version();
235 }
236
237 public void setVersionNo(final Long version) {
238 asDocMeta().version(version);
239 }
240
241 @Override
242 public String toString() {
243 return "WebConfig [available=" + available + ", boost=" + boost + ", configParameter=" + configParameter + ", createdBy="
244 + createdBy + ", createdTime=" + createdTime + ", depth=" + depth + ", excludedDocUrls=" + excludedDocUrls
245 + ", excludedUrls=" + excludedUrls + ", includedDocUrls=" + includedDocUrls + ", includedUrls=" + includedUrls
246 + ", intervalTime=" + intervalTime + ", timeToLive=" + timeToLive + ", maxAccessCount=" + maxAccessCount + ", name=" + name
247 + ", numOfThread=" + numOfThread + ", permissions=" + Arrays.toString(permissions) + ", sortOrder=" + sortOrder
248 + ", updatedBy=" + updatedBy + ", updatedTime=" + updatedTime + ", urls=" + urls + ", userAgent=" + userAgent + "]";
249 }
250 }