1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.opensearch.config.exentity;
17
18 import java.util.ArrayList;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.function.Supplier;
25 import java.util.regex.Pattern;
26
27 import org.apache.logging.log4j.LogManager;
28 import org.apache.logging.log4j.Logger;
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.fess.Constants;
31 import org.codelibs.fess.app.service.FileAuthenticationService;
32 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
33 import org.codelibs.fess.crawler.client.ftp.FtpAuthentication;
34 import org.codelibs.fess.crawler.client.smb.SmbAuthentication;
35 import org.codelibs.fess.helper.SystemHelper;
36 import org.codelibs.fess.opensearch.config.bsentity.BsFileConfig;
37 import org.codelibs.fess.util.ComponentUtil;
38 import org.codelibs.fess.util.ParameterUtil;
39
40
41
42
43 public class FileConfig extends BsFileConfig implements CrawlingConfig {
44
45 private static final Logger logger = LogManager.getLogger(FileConfig.class);
46
47 private static final long serialVersionUID = 1L;
48
49 protected volatile Pattern[] includedDocPathPatterns;
50
51 protected volatile Pattern[] excludedDocPathPatterns;
52
53 protected transient volatile Map<ConfigName, Map<String, String>> configParameterMap;
54
55 protected CrawlerClientFactory crawlerClientFactory = null;
56
57 public FileConfig() {
58 setBoost(1.0f);
59 }
60
61 @Override
62 public String getDocumentBoost() {
63 return getBoost().toString();
64 }
65
66 @Override
67 public String getIndexingTarget(final String input) {
68 if (includedDocPathPatterns == null || excludedDocPathPatterns == null) {
69 initDocPathPattern();
70 }
71
72 if (includedDocPathPatterns.length == 0 && excludedDocPathPatterns.length == 0) {
73 return Constants.TRUE;
74 }
75
76 for (final Pattern pattern : includedDocPathPatterns) {
77 if (pattern.matcher(input).matches()) {
78 return Constants.TRUE;
79 }
80 }
81
82 for (final Pattern pattern : excludedDocPathPatterns) {
83 if (pattern.matcher(input).matches()) {
84 return Constants.FALSE;
85 }
86 }
87
88 return Constants.TRUE;
89
90 }
91
92 protected synchronized void initDocPathPattern() {
93 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
94
95 if (includedDocPathPatterns == null) {
96 if (StringUtil.isNotBlank(getIncludedDocPaths())) {
97 final List<Pattern> pathPatterList = new ArrayList<>();
98 final String[] paths = getIncludedDocPaths().split("[\r\n]");
99 for (final String u : paths) {
100 final String v = systemHelper.normalizeConfigPath(u);
101 if (StringUtil.isNotBlank(v)) {
102 pathPatterList.add(Pattern.compile(systemHelper.encodeUrlFilter(v)));
103 }
104 }
105 includedDocPathPatterns = pathPatterList.toArray(new Pattern[pathPatterList.size()]);
106 } else {
107 includedDocPathPatterns = new Pattern[0];
108 }
109 }
110
111 if (excludedDocPathPatterns == null) {
112 if (StringUtil.isNotBlank(getExcludedDocPaths())) {
113 final List<Pattern> pathPatterList = new ArrayList<>();
114 final String[] paths = getExcludedDocPaths().split("[\r\n]");
115 for (final String u : paths) {
116 final String v = systemHelper.normalizeConfigPath(u);
117 if (StringUtil.isNotBlank(v)) {
118 pathPatterList.add(Pattern.compile(systemHelper.encodeUrlFilter(v)));
119 }
120 }
121 excludedDocPathPatterns = pathPatterList.toArray(new Pattern[pathPatterList.size()]);
122 } else if (includedDocPathPatterns.length > 0) {
123 excludedDocPathPatterns = new Pattern[] { Pattern.compile(".*") };
124 } else {
125 excludedDocPathPatterns = new Pattern[0];
126 }
127 }
128 }
129
130 public String getBoostValue() {
131 if (boost != null) {
132 return boost.toString();
133 }
134 return null;
135 }
136
137 public void setBoostValue(final String value) {
138 if (value != null) {
139 try {
140 boost = Float.parseFloat(value);
141 } catch (final Exception e) {
142 if (logger.isDebugEnabled()) {
143 logger.debug("Invalid boost value, keeping current: value={}, error={}", value, e.getMessage());
144 }
145 }
146 }
147 }
148
149 @Override
150 public String getConfigId() {
151 return ConfigType.FILE.getConfigId(getId());
152 }
153
154 @Override
155 public CrawlerClientFactory initializeClientFactory(final Supplier<CrawlerClientFactory> creator) {
156 if (crawlerClientFactory != null) {
157 return crawlerClientFactory;
158 }
159 final CrawlerClientFactory factory = creator.get();
160
161 final FileAuthenticationService fileAuthenticationService = ComponentUtil.getComponent(FileAuthenticationService.class);
162
163
164 final Map<String, Object> paramMap = new HashMap<>();
165 factory.setInitParameterMap(paramMap);
166
167 final Map<String, String> clientConfigMap = getConfigParameterMap(ConfigName.CLIENT);
168 if (clientConfigMap != null) {
169 paramMap.putAll(clientConfigMap);
170 }
171
172
173 final List<FileAuthentication> fileAuthList = fileAuthenticationService.getFileAuthenticationList(getId());
174 final List<SmbAuthentication> smbAuthList = new ArrayList<>();
175 final List<org.codelibs.fess.crawler.client.smb1.SmbAuthentication> smb1AuthList = new ArrayList<>();
176 final List<FtpAuthentication> ftpAuthList = new ArrayList<>();
177 for (final FileAuthentication fileAuth : fileAuthList) {
178 if (logger.isDebugEnabled()) {
179 logger.debug("FileAuthentication: " + fileAuth.getProtocolScheme() + " " + fileAuth.getHostname() + ":" + fileAuth.getPort()
180 + " " + fileAuth.getUsername());
181 }
182 if (Constants.SAMBA.equals(fileAuth.getProtocolScheme())) {
183 final SmbAuthentication smbAuth = new SmbAuthentication();
184 final Map<String, String> map = ParameterUtil.parse(fileAuth.getParameters());
185 final String domain = map.get("domain");
186 smbAuth.setDomain(domain == null ? StringUtil.EMPTY : domain);
187 smbAuth.setServer(fileAuth.getHostname());
188 smbAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
189 smbAuth.setUsername(fileAuth.getUsername());
190 smbAuth.setPassword(fileAuth.getPassword());
191 smbAuthList.add(smbAuth);
192
193 final org.codelibs.fess.crawler.client.smb1.SmbAuthentication smb1Auth =
194 new org.codelibs.fess.crawler.client.smb1.SmbAuthentication();
195 smb1Auth.setDomain(domain == null ? StringUtil.EMPTY : domain);
196 smb1Auth.setServer(fileAuth.getHostname());
197 smb1Auth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
198 smb1Auth.setUsername(fileAuth.getUsername());
199 smb1Auth.setPassword(fileAuth.getPassword());
200 smb1AuthList.add(smb1Auth);
201 } else if (Constants.FTP.equals(fileAuth.getProtocolScheme())) {
202 final FtpAuthentication ftpAuth = new FtpAuthentication();
203 ftpAuth.setServer(fileAuth.getHostname());
204 ftpAuth.setPort(fileAuth.getPort() == null ? -1 : fileAuth.getPort());
205 ftpAuth.setUsername(fileAuth.getUsername());
206 ftpAuth.setPassword(fileAuth.getPassword());
207 ftpAuthList.add(ftpAuth);
208 }
209 }
210 paramMap.put(Param.Client.SMB_AUTHENTICATIONS, smbAuthList.toArray(new SmbAuthentication[smbAuthList.size()]));
211 if (logger.isDebugEnabled()) {
212 smbAuthList.forEach(smbAuth -> logger
213 .debug("SmbAuthentication: " + smbAuth.getServer() + ":" + smbAuth.getPort() + " " + smbAuth.getUsername()));
214 }
215 paramMap.put(Param.Client.SMB1_AUTHENTICATIONS,
216 smb1AuthList.toArray(new org.codelibs.fess.crawler.client.smb1.SmbAuthentication[smb1AuthList.size()]));
217 if (logger.isDebugEnabled()) {
218 smb1AuthList.forEach(smb1Auth -> logger
219 .debug("Smb1Authentication: " + smb1Auth.getServer() + ":" + smb1Auth.getPort() + " " + smb1Auth.getUsername()));
220 }
221 paramMap.put(Param.Client.FTP_AUTHENTICATIONS, ftpAuthList.toArray(new FtpAuthentication[ftpAuthList.size()]));
222 if (logger.isDebugEnabled()) {
223 ftpAuthList.forEach(ftpAuth -> logger
224 .debug("FtpAuthentication: " + ftpAuth.getServer() + ":" + ftpAuth.getPort() + " " + ftpAuth.getUsername()));
225 }
226
227 crawlerClientFactory = factory;
228 return factory;
229 }
230
231 @Override
232 public Map<String, String> getConfigParameterMap(final ConfigName name) {
233 if (configParameterMap == null) {
234 configParameterMap = ParameterUtil.createConfigParameterMap(getConfigParameter());
235 }
236
237 final Map<String, String> configMap = configParameterMap.get(name);
238 if (configMap == null) {
239 return Collections.emptyMap();
240 }
241 return configMap;
242 }
243
244 @Override
245 public String getId() {
246 return asDocMeta().id();
247 }
248
249 public void setId(final String id) {
250 asDocMeta().id(id);
251 }
252
253 public Long getVersionNo() {
254 return asDocMeta().version();
255 }
256
257 public void setVersionNo(final Long version) {
258 asDocMeta().version(version);
259 }
260
261 @Override
262 public String toString() {
263 return "FileConfig [available=" + available + ", boost=" + boost + ", configParameter=" + configParameter + ", createdBy="
264 + createdBy + ", createdTime=" + createdTime + ", depth=" + depth + ", excludedDocPaths=" + excludedDocPaths
265 + ", excludedPaths=" + excludedPaths + ", includedDocPaths=" + includedDocPaths + ", includedPaths=" + includedPaths
266 + ", intervalTime=" + intervalTime + ", timeToLive=" + timeToLive + ", maxAccessCount=" + maxAccessCount + ", name=" + name
267 + ", numOfThread=" + numOfThread + ", paths=" + paths + ", permissions=" + Arrays.toString(permissions) + ", sortOrder="
268 + sortOrder + ", updatedBy=" + updatedBy + ", updatedTime=" + updatedTime + "]";
269 }
270 }