1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.opensearch.config.exentity;
17
18 import java.util.Map;
19 import java.util.function.Supplier;
20
21 import org.apache.http.auth.UsernamePasswordCredentials;
22 import org.codelibs.core.lang.StringUtil;
23 import org.codelibs.fess.Constants;
24 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
25 import org.codelibs.fess.crawler.client.ftp.FtpClient;
26 import org.codelibs.fess.crawler.client.http.HcHttpClient;
27 import org.codelibs.fess.crawler.client.smb.SmbClient;
28 import org.codelibs.fess.mylasta.direction.FessConfig;
29 import org.codelibs.fess.util.ComponentUtil;
30
31 public interface CrawlingConfig {
32
33 String getId();
34
35 String getName();
36
37 String[] getPermissions();
38
39 String[] getVirtualHosts();
40
41 String getDocumentBoost();
42
43 String getIndexingTarget(String input);
44
45 String getConfigId();
46
47 Integer getTimeToLive();
48
49 CrawlerClientFactory initializeClientFactory(Supplier<CrawlerClientFactory> creator);
50
51 Map<String, String> getConfigParameterMap(ConfigName name);
52
53 default void initializeDefaultHttpProxy(final Map<String, Object> paramMap) {
54 final FessConfig fessConfig = ComponentUtil.getFessConfig();
55 final String proxyHost = fessConfig.getHttpProxyHost();
56 final String proxyPort = fessConfig.getHttpProxyPort();
57 if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
58 paramMap.put(Param.Client.PROXY_HOST, proxyHost);
59 paramMap.put(Param.Client.PROXY_PORT, proxyPort);
60 final String proxyUsername = fessConfig.getHttpProxyUsername();
61 final String proxyPassword = fessConfig.getHttpProxyPassword();
62 if (proxyUsername != null && proxyPassword != null) {
63 paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
64 }
65
66 }
67 }
68
69 default String getScriptType() {
70 final String scriptType = getConfigParameterMap(ConfigName.CONFIG).get(Param.Config.SCRIPT_TYPE);
71 if (StringUtil.isNotBlank(scriptType)) {
72 return scriptType;
73 }
74 return Constants.DEFAULT_SCRIPT;
75 }
76
77 public enum ConfigType {
78 WEB("W"), FILE("F"), DATA("D");
79
80 private final String typePrefix;
81
82 ConfigType(final String typePrefix) {
83 this.typePrefix = typePrefix;
84 }
85
86 public String getTypePrefix() {
87 return typePrefix;
88 }
89
90 String getConfigId(final String id) {
91 if (id == null) {
92 return null;
93 }
94 return typePrefix + id.toString();
95 }
96 }
97
98 public enum ConfigName {
99 CLIENT, XPATH, META, VALUE, SCRIPT, FIELD, CONFIG;
100 }
101
102 public static class Param {
103
104 public static class Client {
105 public static final String SMB_AUTHENTICATIONS = SmbClient.SMB_AUTHENTICATIONS_PROPERTY;
106 public static final String SMB1_AUTHENTICATIONS = org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY;
107 public static final String FTP_AUTHENTICATIONS = FtpClient.FTP_AUTHENTICATIONS_PROPERTY;
108 public static final String ROBOTS_TXT_ENABLED = HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY;
109 public static final String PROXY_PASSWORD = "proxyPassword";
110 public static final String PROXY_USERNAME = "proxyUsername";
111 public static final String PROXY_PORT = HcHttpClient.PROXY_PORT_PROPERTY;
112 public static final String PROXY_HOST = HcHttpClient.PROXY_HOST_PROPERTY;
113 public static final String USER_AGENT = HcHttpClient.USER_AGENT_PROPERTY;
114 }
115
116
117 public static class XPath {
118 public static final String DEFAULT_LANG = "default.lang";
119 public static final String DEFAULT_CONTENT = "default.content";
120 public static final String DEFAULT_DIGEST = "default.digest";
121
122 }
123
124
125 public static class Config {
126 public static final String KEEP_ORIGINAL_BODY = "keep.original.body";
127 public static final String CLEANUP_ALL = "cleanup.all";
128 public static final String CLEANUP_URL_FILTERS = "cleanup.urlFilters";
129 public static final String JCIFS_PREFIX = "jcifs.";
130 public static final String HTML_CANONICAL_XPATH = "html.canonical.xpath";
131 public static final String HTML_PRUNED_TAGS = "html.pruned.tags";
132 public static final String PIPELINE = "pipeline";
133 public static final String IGNORE_ROBOTS_TAGS = "ignore.robots.tags";
134 public static final String SCRIPT_TYPE = "script.type";
135 public static final String HTML_CHILD_URL_RULES = "html.child.url.rules";
136 public static final String CRAWL_ORDER = "crawl.order";
137 }
138
139
140
141
142
143
144
145
146
147
148
149
150 }
151 }