1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.opensearch.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.function.Supplier;
25 import java.util.regex.Pattern;
26
27 import org.apache.http.auth.UsernamePasswordCredentials;
28 import org.apache.logging.log4j.LogManager;
29 import org.apache.logging.log4j.Logger;
30 import org.codelibs.core.lang.StringUtil;
31 import org.codelibs.fess.Constants;
32 import org.codelibs.fess.app.service.RequestHeaderService;
33 import org.codelibs.fess.app.service.WebAuthenticationService;
34 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
35 import org.codelibs.fess.crawler.client.http.HcHttpClient;
36 import org.codelibs.fess.crawler.client.http.config.WebAuthenticationConfig;
37 import org.codelibs.fess.helper.SystemHelper;
38 import org.codelibs.fess.mylasta.direction.FessConfig;
39 import org.codelibs.fess.opensearch.config.bsentity.BsWebConfig;
40 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.Param.Client;
41 import org.codelibs.fess.util.ComponentUtil;
42 import org.codelibs.fess.util.ParameterUtil;
43
44
45
46
47 public class WebConfig extends BsWebConfig implements CrawlingConfig {
48
49 private static final Logger logger = LogManager.getLogger(WebConfig.class);
50
51 private static final long serialVersionUID = 1L;
52
53 protected volatile Pattern[] includedDocUrlPatterns;
54
55 protected volatile Pattern[] excludedDocUrlPatterns;
56
57 protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
58
59 protected CrawlerClientFactory crawlerClientFactory = null;
60
61 public WebConfig() {
62 setBoost(1.0f);
63 }
64
65 @Override
66 public String getDocumentBoost() {
67 return getBoost().toString();
68 }
69
70 @Override
71 public String getIndexingTarget(final String input) {
72 if (includedDocUrlPatterns == null || excludedDocUrlPatterns == null) {
73 initDocUrlPattern();
74 }
75
76 if (includedDocUrlPatterns.length == 0 && excludedDocUrlPatterns.length == 0) {
77 return Constants.TRUE;
78 }
79
80 for (final Pattern pattern : includedDocUrlPatterns) {
81 if (pattern.matcher(input).matches()) {
82 return Constants.TRUE;
83 }
84 }
85
86 for (final Pattern pattern : excludedDocUrlPatterns) {
87 if (pattern.matcher(input).matches()) {
88 return Constants.FALSE;
89 }
90 }
91
92 return Constants.TRUE;
93 }
94
95 protected synchronized void initDocUrlPattern() {
96
97 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
98 if (includedDocUrlPatterns == null) {
99 if (StringUtil.isNotBlank(getIncludedDocUrls())) {
100 final List<Pattern> urlPatterList = new ArrayList<>();
101 final String[] urls = getIncludedDocUrls().split("[\r\n]");
102 for (final String u : urls) {
103 final String v = systemHelper.normalizeConfigPath(u);
104 if (StringUtil.isNotBlank(v)) {
105 urlPatterList.add(Pattern.compile(v));
106 }
107 }
108 includedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
109 } else {
110 includedDocUrlPatterns = new Pattern[0];
111 }
112 }
113
114 if (excludedDocUrlPatterns == null) {
115 if (StringUtil.isNotBlank(getExcludedDocUrls())) {
116 final List<Pattern> urlPatterList = new ArrayList<>();
117 final String[] urls = getExcludedDocUrls().split("[\r\n]");
118 for (final String u : urls) {
119 final String v = systemHelper.normalizeConfigPath(u);
120 if (StringUtil.isNotBlank(v)) {
121 urlPatterList.add(Pattern.compile(v));
122 }
123 }
124 excludedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
125 } else if (includedDocUrlPatterns.length > 0) {
126 excludedDocUrlPatterns = new Pattern[] { Pattern.compile(".*") };
127 } else {
128 excludedDocUrlPatterns = new Pattern[0];
129 }
130 }
131 }
132
133 public String getBoostValue() {
134 if (boost != null) {
135 return boost.toString();
136 }
137 return null;
138 }
139
140 public void setBoostValue(final String value) {
141 if (value != null) {
142 try {
143 boost = Float.parseFloat(value);
144 } catch (final Exception e) {
145 if (logger.isDebugEnabled()) {
146 logger.debug("Invalid boost value, keeping current: value={}, error={}", value, e.getMessage());
147 }
148 }
149 }
150 }
151
152 @Override
153 public String getConfigId() {
154 return ConfigType.WEB.getConfigId(getId());
155 }
156
157 @Override
158 public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
159 if (crawlerClientFactory != null) {
160 return crawlerClientFactory;
161 }
162 final CrawlerClientFactory factory = creator.get();
163
164 final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
165 final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
166 final FessConfig fessConfig = ComponentUtil.getFessConfig();
167
168
169 final Map<String, Object> paramMap = new HashMap<>();
170 factory.setInitParameterMap(paramMap);
171
172 final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
173 if (clientConfigMap != null) {
174 paramMap.putAll(clientConfigMap);
175 }
176
177
178 if (paramMap.get(Param.Client.ROBOTS_TXT_ENABLED) == null) {
179 paramMap.put(Param.Client.ROBOTS_TXT_ENABLED, !fessConfig.isCrawlerIgnoreRobotsTxt());
180 }
181
182 final String userAgent = getUserAgent();
183 if (!StringUtil.isNotBlank(userAgent) || (userAgent.startsWith(Constants.CRAWLING_USER_AGENT_PREFIX)
184 && userAgent.endsWith(Constants.CRAWLING_USER_AGENT_SUFFIX))) {
185 paramMap.put(Client.USER_AGENT, fessConfig.getUserAgentName());
186 } else {
187 paramMap.put(Client.USER_AGENT, userAgent);
188 }
189
190 final List<WebAuthentication> webAuthList = webAuthenticationService.getWebAuthenticationList(getId());
191 final List<WebAuthenticationConfig> authConfigList = new ArrayList<>();
192 for (final WebAuthentication webAuth : webAuthList) {
193 authConfigList.add(webAuth.getWebAuthenticationConfig());
194 }
195 paramMap.put(HcHttpClient.AUTHENTICATIONS_PROPERTY, authConfigList.toArray(new WebAuthenticationConfig[authConfigList.size()]));
196
197
198 final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
199 final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
200 for (final RequestHeader requestHeader : requestHeaderList) {
201 rhList.add(requestHeader.getCrawlerRequestHeader());
202 }
203 paramMap.put(HcHttpClient.REQUEST_HEADERS_PROPERTY,
204 rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
205
206 final String proxyHost = (String) paramMap.get(Param.Client.PROXY_HOST);
207 final String proxyPort = (String) paramMap.get(Param.Client.PROXY_PORT);
208 if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
209
210 if (paramMap.get(Param.Client.PROXY_USERNAME) != null && paramMap.get(Param.Client.PROXY_PASSWORD) != null) {
211 paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(
212 paramMap.remove(Param.Client.PROXY_USERNAME).toString(), paramMap.remove(Param.Client.PROXY_PASSWORD).toString()));
213 }
214 } else {
215 initializeDefaultHttpProxy(paramMap);
216 }
217
218 crawlerClientFactory = factory;
219 return factory;
220 }
221
222 @Override
223 public Map<String, String> getConfigParameterMap(final ConfigName name) {
224 if (configParameterMap == null) {
225 configParameterMap = ParameterUtil.createConfigParameterMap(getConfigParameter());
226 }
227
228 final Map<String, String> configMap = configParameterMap.get(name);
229 if (configMap == null) {
230 return Collections.emptyMap();
231 }
232 return configMap;
233 }
234
235 @Override
236 public String getId() {
237 return asDocMeta().id();
238 }
239
240 public void setId(final String id) {
241 asDocMeta().id(id);
242 }
243
244 public Long getVersionNo() {
245 return asDocMeta().version();
246 }
247
248 public void setVersionNo(final Long version) {
249 asDocMeta().version(version);
250 }
251
252 @Override
253 public String toString() {
254 return "WebConfig [available=" + available + ", boost=" + boost + ", configParameter=" + configParameter + ", createdBy="
255 + createdBy + ", createdTime=" + createdTime + ", depth=" + depth + ", excludedDocUrls=" + excludedDocUrls
256 + ", excludedUrls=" + excludedUrls + ", includedDocUrls=" + includedDocUrls + ", includedUrls=" + includedUrls
257 + ", intervalTime=" + intervalTime + ", timeToLive=" + timeToLive + ", maxAccessCount=" + maxAccessCount + ", name=" + name
258 + ", numOfThread=" + numOfThread + ", permissions=" + Arrays.toString(permissions) + ", sortOrder=" + sortOrder
259 + ", updatedBy=" + updatedBy + ", updatedTime=" + updatedTime + ", urls=" + urls + ", userAgent=" + userAgent + "]";
260 }
261 }