1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.es.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.regex.Pattern;
25
26 import org.apache.http.auth.UsernamePasswordCredentials;
27 import org.codelibs.core.lang.StringUtil;
28 import org.codelibs.fess.Constants;
29 import org.codelibs.fess.app.service.RequestHeaderService;
30 import org.codelibs.fess.app.service.WebAuthenticationService;
31 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
32 import org.codelibs.fess.crawler.client.http.Authentication;
33 import org.codelibs.fess.crawler.client.http.HcHttpClient;
34 import org.codelibs.fess.es.config.bsentity.BsWebConfig;
35 import org.codelibs.fess.es.config.exbhv.LabelTypeBhv;
36 import org.codelibs.fess.es.config.exbhv.WebConfigToLabelBhv;
37 import org.codelibs.fess.mylasta.direction.FessConfig;
38 import org.codelibs.fess.util.ComponentUtil;
39 import org.codelibs.fess.util.ParameterUtil;
40 import org.dbflute.cbean.result.ListResultBean;
41
42
43
44
45 public class WebConfig extends BsWebConfig implements CrawlingConfig {
46
47 private static final long serialVersionUID = 1L;
48
49 private String[] labelTypeIds;
50
51 protected volatile Pattern[] includedDocUrlPatterns;
52
53 protected volatile Pattern[] excludedDocUrlPatterns;
54
55 protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
56
57 private volatile List<LabelType> labelTypeList;
58
59 public WebConfig() {
60 super();
61 setBoost(1.0f);
62 }
63
64
65
66
67 public String[] getLabelTypeIds() {
68 if (labelTypeIds == null) {
69 return StringUtil.EMPTY_STRINGS;
70 }
71 return labelTypeIds;
72 }
73
74 public void setLabelTypeIds(final String[] labelTypeIds) {
75 this.labelTypeIds = labelTypeIds;
76 }
77
78 public List<LabelType> getLabelTypeList() {
79 if (labelTypeList == null) {
80 synchronized (this) {
81 if (labelTypeList == null) {
82 final FessConfig fessConfig = ComponentUtil.getFessConfig();
83 final WebConfigToLabelBhv webConfigToLabelBhv = ComponentUtil.getComponent(WebConfigToLabelBhv.class);
84 final ListResultBean<WebConfigToLabel> mappingList = webConfigToLabelBhv.selectList(cb -> {
85 cb.query().setWebConfigId_Equal(getId());
86 cb.specify().columnLabelTypeId();
87 cb.paging(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger().intValue(), 1);
88 });
89 final List<String> labelIdList = new ArrayList<>();
90 for (final WebConfigToLabel mapping : mappingList) {
91 labelIdList.add(mapping.getLabelTypeId());
92 }
93 final LabelTypeBhv labelTypeBhv = ComponentUtil.getComponent(LabelTypeBhv.class);
94 labelTypeList = labelIdList.isEmpty() ? Collections.emptyList() : labelTypeBhv.selectList(cb -> {
95 cb.query().setId_InScope(labelIdList);
96 cb.query().addOrderBy_SortOrder_Asc();
97 cb.fetchFirst(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger());
98 });
99 }
100 }
101 }
102 return labelTypeList;
103 }
104
105 @Override
106 public String[] getLabelTypeValues() {
107 final List<LabelType> list = getLabelTypeList();
108 final List<String> labelValueList = new ArrayList<>(list.size());
109 for (final LabelType labelType : list) {
110 labelValueList.add(labelType.getValue());
111 }
112 return labelValueList.toArray(new String[labelValueList.size()]);
113 }
114
115 @Override
116 public String getDocumentBoost() {
117 return Float.valueOf(getBoost().floatValue()).toString();
118 }
119
120 @Override
121 public String getIndexingTarget(final String input) {
122 if (includedDocUrlPatterns == null || excludedDocUrlPatterns == null) {
123 initDocUrlPattern();
124 }
125
126 if (includedDocUrlPatterns.length == 0 && excludedDocUrlPatterns.length == 0) {
127 return Constants.TRUE;
128 }
129
130 for (final Pattern pattern : includedDocUrlPatterns) {
131 if (pattern.matcher(input).matches()) {
132 return Constants.TRUE;
133 }
134 }
135
136 for (final Pattern pattern : excludedDocUrlPatterns) {
137 if (pattern.matcher(input).matches()) {
138 return Constants.FALSE;
139 }
140 }
141
142 return Constants.TRUE;
143 }
144
145 protected synchronized void initDocUrlPattern() {
146
147 if (includedDocUrlPatterns == null) {
148 if (StringUtil.isNotBlank(getIncludedDocUrls())) {
149 final List<Pattern> urlPatterList = new ArrayList<>();
150 final String[] urls = getIncludedDocUrls().split("[\r\n]");
151 for (final String u : urls) {
152 if (StringUtil.isNotBlank(u) && !u.trim().startsWith("#")) {
153 urlPatterList.add(Pattern.compile(u.trim()));
154 }
155 }
156 includedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
157 } else {
158 includedDocUrlPatterns = new Pattern[0];
159 }
160 }
161
162 if (excludedDocUrlPatterns == null) {
163 if (StringUtil.isNotBlank(getExcludedDocUrls())) {
164 final List<Pattern> urlPatterList = new ArrayList<>();
165 final String[] urls = getExcludedDocUrls().split("[\r\n]");
166 for (final String u : urls) {
167 if (StringUtil.isNotBlank(u) && !u.trim().startsWith("#")) {
168 urlPatterList.add(Pattern.compile(u.trim()));
169 }
170 }
171 excludedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
172 } else if (includedDocUrlPatterns.length > 0) {
173 excludedDocUrlPatterns = new Pattern[] { Pattern.compile(".*") };
174 } else {
175 excludedDocUrlPatterns = new Pattern[0];
176 }
177 }
178 }
179
180 public String getBoostValue() {
181 if (boost != null) {
182 return boost.toString();
183 }
184 return null;
185 }
186
187 public void setBoostValue(final String value) {
188 if (value != null) {
189 try {
190 boost = Float.parseFloat(value);
191 } catch (final Exception e) {}
192 }
193 }
194
195 @Override
196 public String getConfigId() {
197 return ConfigType.WEB.getConfigId(getId());
198 }
199
200 @Override
201 public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
202 final WebAuthenticationService webAuthenticationService = ComponentUtil.getComponent(WebAuthenticationService.class);
203 final RequestHeaderService requestHeaderService = ComponentUtil.getComponent(RequestHeaderService.class);
204 final FessConfig fessConfig = ComponentUtil.getFessConfig();
205
206
207 final Map<String, Object> paramMap = new HashMap<>();
208 clientFactory.setInitParameterMap(paramMap);
209
210 final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
211 if (clientConfigMap != null) {
212 paramMap.putAll(clientConfigMap);
213 }
214
215
216 if (paramMap.get(HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY) == null) {
217 paramMap.put(HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY, !fessConfig.isCrawlerIgnoreRobotsTxt());
218 }
219
220 final String userAgent = getUserAgent();
221 if (StringUtil.isNotBlank(userAgent)) {
222 paramMap.put(HcHttpClient.USER_AGENT_PROPERTY, userAgent);
223 }
224
225 final List<WebAuthentication> webAuthList = webAuthenticationService.getWebAuthenticationList(getId());
226 final List<Authentication> basicAuthList = new ArrayList<>();
227 for (final WebAuthentication webAuth : webAuthList) {
228 basicAuthList.add(webAuth.getAuthentication());
229 }
230 paramMap.put(HcHttpClient.BASIC_AUTHENTICATIONS_PROPERTY, basicAuthList.toArray(new Authentication[basicAuthList.size()]));
231
232
233 final List<RequestHeader> requestHeaderList = requestHeaderService.getRequestHeaderList(getId());
234 final List<org.codelibs.fess.crawler.client.http.RequestHeader> rhList = new ArrayList<>();
235 for (final RequestHeader requestHeader : requestHeaderList) {
236 rhList.add(requestHeader.getCrawlerRequestHeader());
237 }
238 paramMap.put(HcHttpClient.REQUERT_HEADERS_PROPERTY,
239 rhList.toArray(new org.codelibs.fess.crawler.client.http.RequestHeader[rhList.size()]));
240
241
242 if (paramMap.get("proxyUsername") != null && paramMap.get("proxyPassword") != null) {
243 paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(paramMap.remove("proxyUsername")
244 .toString(), paramMap.remove("proxyPassword").toString()));
245 }
246
247 return paramMap;
248 }
249
250 @Override
251 public Map<String, String> getConfigParameterMap(final ConfigName name) {
252 if (configParameterMap == null) {
253 configParameterMap = ParameterUtil.createConfigParameterMap(getConfigParameter());
254 }
255
256 final Map<String, String> configMap = configParameterMap.get(name);
257 if (configMap == null) {
258 return Collections.emptyMap();
259 }
260 return configMap;
261 }
262
263 @Override
264 public String getId() {
265 return asDocMeta().id();
266 }
267
268 public void setId(final String id) {
269 asDocMeta().id(id);
270 }
271
272 public Long getVersionNo() {
273 return asDocMeta().version();
274 }
275
276 public void setVersionNo(final Long version) {
277 asDocMeta().version(version);
278 }
279
280 @Override
281 public String toString() {
282 return "WebConfig [available=" + available + ", boost=" + boost + ", configParameter=" + configParameter + ", createdBy="
283 + createdBy + ", createdTime=" + createdTime + ", depth=" + depth + ", excludedDocUrls=" + excludedDocUrls
284 + ", excludedUrls=" + excludedUrls + ", includedDocUrls=" + includedDocUrls + ", includedUrls=" + includedUrls
285 + ", intervalTime=" + intervalTime + ", timeToLive=" + timeToLive + ", maxAccessCount=" + maxAccessCount + ", name=" + name
286 + ", numOfThread=" + numOfThread + ", permissions=" + Arrays.toString(permissions) + ", sortOrder=" + sortOrder
287 + ", updatedBy=" + updatedBy + ", updatedTime=" + updatedTime + ", urls=" + urls + ", userAgent=" + userAgent + "]";
288 }
289
290 }