1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.es.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.regex.Pattern;
25
26 import org.codelibs.core.lang.StringUtil;
27 import org.codelibs.fess.Constants;
28 import org.codelibs.fess.app.service.FileAuthenticationService;
29 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
30 import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
31 import org.codelibs.fess.crawler.client.ftp.FtpClient;
32 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
33 import org.codelibs.fess.crawler.client.smb.SmbClient;
34 import org.codelibs.fess.es.config.bsentity.BsFileConfig;
35 import org.codelibs.fess.es.config.exbhv.FileConfigToLabelBhv;
36 import org.codelibs.fess.es.config.exbhv.LabelTypeBhv;
37 import org.codelibs.fess.helper.SystemHelper;
38 import org.codelibs.fess.mylasta.direction.FessConfig;
39 import org.codelibs.fess.util.ComponentUtil;
40 import org.codelibs.fess.util.ParameterUtil;
41 import org.dbflute.cbean.result.ListResultBean;
42
43
44
45
46 public class FileConfig extends BsFileConfig implements CrawlingConfig {
47
48 private static final long serialVersionUID = 1L;
49
50 private String[] labelTypeIds;
51
52 protected volatile Pattern[] includedDocPathPatterns;
53
54 protected volatile Pattern[] excludedDocPathPatterns;
55
56 protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
57
58 private volatile List<LabelType> labelTypeList;
59
60 public FileConfig() {
61 super();
62 setBoost(1.0f);
63 }
64
65 public String[] getLabelTypeIds() {
66 if (labelTypeIds == null) {
67 return StringUtil.EMPTY_STRINGS;
68 }
69 return labelTypeIds;
70 }
71
72 public void setLabelTypeIds(final String[] labelTypeIds) {
73 this.labelTypeIds = labelTypeIds;
74 }
75
76 public List<LabelType> getLabelTypeList() {
77 if (labelTypeList == null) {
78 synchronized (this) {
79 if (labelTypeList == null) {
80 final FessConfig fessConfig = ComponentUtil.getFessConfig();
81 final FileConfigToLabelBhv fileConfigToLabelBhv = ComponentUtil.getComponent(FileConfigToLabelBhv.class);
82 final ListResultBean<FileConfigToLabel> mappingList = fileConfigToLabelBhv.selectList(cb -> {
83 cb.query().setFileConfigId_Equal(getId());
84 cb.specify().columnLabelTypeId();
85 cb.paging(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger().intValue(), 1);
86 });
87 final List<String> labelIdList = new ArrayList<>();
88 for (final FileConfigToLabel mapping : mappingList) {
89 labelIdList.add(mapping.getLabelTypeId());
90 }
91 final LabelTypeBhv labelTypeBhv = ComponentUtil.getComponent(LabelTypeBhv.class);
92 labelTypeList = labelIdList.isEmpty() ? Collections.emptyList() : labelTypeBhv.selectList(cb -> {
93 cb.query().setId_InScope(labelIdList);
94 cb.query().addOrderBy_SortOrder_Asc();
95 cb.fetchFirst(fessConfig.getPageLabeltypeMaxFetchSizeAsInteger());
96 });
97 }
98 }
99 }
100 return labelTypeList;
101 }
102
103 @Override
104 public String[] getLabelTypeValues() {
105 final List<LabelType> list = getLabelTypeList();
106 final List<String> labelValueList = new ArrayList<>(list.size());
107 for (final LabelType labelType : list) {
108 labelValueList.add(labelType.getValue());
109 }
110 return labelValueList.toArray(new String[labelValueList.size()]);
111 }
112
113 @Override
114 public String getDocumentBoost() {
115 return Float.valueOf(getBoost().floatValue()).toString();
116 }
117
118 @Override
119 public String getIndexingTarget(final String input) {
120 if (includedDocPathPatterns == null || excludedDocPathPatterns == null) {
121 initDocPathPattern();
122 }
123
124 if (includedDocPathPatterns.length == 0 && excludedDocPathPatterns.length == 0) {
125 return Constants.TRUE;
126 }
127
128 for (final Pattern pattern : includedDocPathPatterns) {
129 if (pattern.matcher(input).matches()) {
130 return Constants.TRUE;
131 }
132 }
133
134 for (final Pattern pattern : excludedDocPathPatterns) {
135 if (pattern.matcher(input).matches()) {
136 return Constants.FALSE;
137 }
138 }
139
140 return Constants.TRUE;
141
142 }
143
144 protected synchronized void initDocPathPattern() {
145 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
146
147 if (includedDocPathPatterns == null) {
148 if (StringUtil.isNotBlank(getIncludedDocPaths())) {
149 final List<Pattern> pathPatterList = new ArrayList<>();
150 final String[] paths = getIncludedDocPaths().split("[\r\n]");
151 for (final String u : paths) {
152 if (StringUtil.isNotBlank(u) && !u.trim().startsWith("#")) {
153 pathPatterList.add(Pattern.compile(systemHelper.encodeUrlFilter(u.trim())));
154 }
155 }
156 includedDocPathPatterns = pathPatterList.toArray(new Pattern[pathPatterList.size()]);
157 } else {
158 includedDocPathPatterns = new Pattern[0];
159 }
160 }
161
162 if (excludedDocPathPatterns == null) {
163 if (StringUtil.isNotBlank(getExcludedDocPaths())) {
164 final List<Pattern> pathPatterList = new ArrayList<>();
165 final String[] paths = getExcludedDocPaths().split("[\r\n]");
166 for (final String u : paths) {
167 if (StringUtil.isNotBlank(u) && !u.trim().startsWith("#")) {
168 pathPatterList.add(Pattern.compile(systemHelper.encodeUrlFilter(u.trim())));
169 }
170 }
171 excludedDocPathPatterns = pathPatterList.toArray(new Pattern[pathPatterList.size()]);
172 } else if (includedDocPathPatterns.length > 0) {
173 excludedDocPathPatterns = new Pattern[] { Pattern.compile(".*") };
174 } else {
175 excludedDocPathPatterns = new Pattern[0];
176 }
177 }
178 }
179
180 public String getBoostValue() {
181 if (boost != null) {
182 return boost.toString();
183 }
184 return null;
185 }
186
187 public void setBoostValue(final String value) {
188 if (value != null) {
189 try {
190 boost = Float.parseFloat(value);
191 } catch (final Exception e) {}
192 }
193 }
194
195 @Override
196 public String getConfigId() {
197 return ConfigType.FILE.getConfigId(getId());
198 }
199
200 @Override
201 public Map<String, Object> initializeClientFactory(final CrawlerClientFactory clientFactory) {
202 final FileAuthenticationService fileAuthenticationService = ComponentUtil.getComponent(FileAuthenticationService.class);
203
204
205 final Map<String, Object> paramMap = new HashMap<>();
206 clientFactory.setInitParameterMap(paramMap);
207
208 final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
209 if (clientConfigMap != null) {
210 paramMap.putAll(clientConfigMap);
211 }
212
213
214 final List<FileAuthentication> fileAuthList = fileAuthenticationService.getFileAuthenticationList(getId());
215 final List<SmbAuthentication> smbAuthList = new ArrayList<>();
216 final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
217 for (final FileAuthentication fileAuth : fileAuthList) {
218 if (Constants.SAMBA.equals(fileAuth.getProtocolScheme())) {
219 final SmbAuthentication smbAuth = new SmbAuthentication();
220 final Map<String, String> map = ParameterUtil.parse(fileAuth.getParameters());
221 final String domain = map.get("domain");
222 smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
223 smbAuth.setServer(fileAuth.getHostname());
224 smbAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort().intValue());
225 smbAuth.setUsername(fileAuth.getUsername());
226 smbAuth.setPassword(fileAuth.getPassword());
227 smbAuthList.add(smbAuth);
228 } else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
229 final FtpAuthentication ftpAuth = new FtpAuthentication();
230 ftpAuth.setServer(fileAuth.getHostname());
231 ftpAuth.setPort(fileAuth.getPort());
232 ftpAuth.setUsername(fileAuth.getUsername());
233 ftpAuth.setPassword(fileAuth.getPassword());
234 ftpAuthList.add(ftpAuth);
235 }
236 }
237 paramMap.put(SmbClient.SMB_AUTHENTICATIONS_PROPERTY, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
238 paramMap.put(FtpClient.FTP_AUTHENTICATIONS_PROPERTY, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
239
240 return paramMap;
241 }
242
243 @Override
244 public Map<String, String> getConfigParameterMap(final ConfigName name) {
245 if (configParameterMap == null) {
246 configParameterMap = ParameterUtil.createConfigParameterMap(getConfigParameter());
247 }
248
249 final Map<String, String> configMap = configParameterMap.get(name);
250 if (configMap == null) {
251 return Collections.emptyMap();
252 }
253 return configMap;
254 }
255
256 @Override
257 public String getId() {
258 return asDocMeta().id();
259 }
260
261 public void setId(final String id) {
262 asDocMeta().id(id);
263 }
264
265 public Long getVersionNo() {
266 return asDocMeta().version();
267 }
268
269 public void setVersionNo(final Long version) {
270 asDocMeta().version(version);
271 }
272
273 @Override
274 public String toString() {
275 return "FileConfig [available=" + available + ", boost=" + boost + ", configParameter=" + configParameter + ", createdBy="
276 + createdBy + ", createdTime=" + createdTime + ", depth=" + depth + ", excludedDocPaths=" + excludedDocPaths
277 + ", excludedPaths=" + excludedPaths + ", includedDocPaths=" + includedDocPaths + ", includedPaths=" + includedPaths
278 + ", intervalTime=" + intervalTime + ", timeToLive=" + timeToLive + ", maxAccessCount=" + maxAccessCount + ", name=" + name
279 + ", numOfThread=" + numOfThread + ", paths=" + paths + ", permissions=" + Arrays.toString(permissions) + ", sortOrder="
280 + sortOrder + ", updatedBy=" + updatedBy + ", updatedTime=" + updatedTime + "]";
281 }
282
283 }