View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.opensearch.config.exentity;
17  
18  import java.util.Map;
19  import java.util.function.Supplier;
20  
21  import org.apache.http.auth.UsernamePasswordCredentials;
22  import org.codelibs.core.lang.StringUtil;
23  import org.codelibs.fess.Constants;
24  import org.codelibs.fess.crawler.client.CrawlerClientFactory;
25  import org.codelibs.fess.crawler.client.ftp.FtpClient;
26  import org.codelibs.fess.crawler.client.http.HcHttpClient;
27  import org.codelibs.fess.crawler.client.smb.SmbClient;
28  import org.codelibs.fess.mylasta.direction.FessConfig;
29  import org.codelibs.fess.util.ComponentUtil;
30  
31  public interface CrawlingConfig {
32  
33      String getId();
34  
35      String getName();
36  
37      String[] getPermissions();
38  
39      String[] getVirtualHosts();
40  
41      String getDocumentBoost();
42  
43      String getIndexingTarget(String input);
44  
45      String getConfigId();
46  
47      Integer getTimeToLive();
48  
49      CrawlerClientFactory initializeClientFactory(Supplier<CrawlerClientFactory> creator);
50  
51      Map<String, String> getConfigParameterMap(ConfigName name);
52  
53      default void initializeDefaultHttpProxy(final Map<String, Object> paramMap) {
54          final FessConfig fessConfig = ComponentUtil.getFessConfig();
55          final String proxyHost = fessConfig.getHttpProxyHost();
56          final String proxyPort = fessConfig.getHttpProxyPort();
57          if (StringUtil.isNotBlank(proxyHost) && StringUtil.isNotBlank(proxyPort)) {
58              paramMap.put(Param.Client.PROXY_HOST, proxyHost);
59              paramMap.put(Param.Client.PROXY_PORT, proxyPort);
60              final String proxyUsername = fessConfig.getHttpProxyUsername();
61              final String proxyPassword = fessConfig.getHttpProxyPassword();
62              if (proxyUsername != null && proxyPassword != null) {
63                  paramMap.put(HcHttpClient.PROXY_CREDENTIALS_PROPERTY, new UsernamePasswordCredentials(proxyUsername, proxyPassword));
64              }
65  
66          }
67      }
68  
69      default String getScriptType() {
70          final String scriptType = getConfigParameterMap(ConfigName.CONFIG).get(Param.Config.SCRIPT_TYPE);
71          if (StringUtil.isNotBlank(scriptType)) {
72              return scriptType;
73          }
74          return Constants.DEFAULT_SCRIPT;
75      }
76  
77      public enum ConfigType {
78          WEB("W"), FILE("F"), DATA("D");
79  
80          private final String typePrefix;
81  
82          ConfigType(final String typePrefix) {
83              this.typePrefix = typePrefix;
84          }
85  
86          public String getTypePrefix() {
87              return typePrefix;
88          }
89  
90          String getConfigId(final String id) {
91              if (id == null) {
92                  return null;
93              }
94              return typePrefix + id.toString();
95          }
96      }
97  
98      public enum ConfigName {
99          CLIENT, XPATH, META, VALUE, SCRIPT, FIELD, CONFIG;
100     }
101 
102     public static class Param {
103         // client.*
104         public static class Client {
105             public static final String SMB_AUTHENTICATIONS = SmbClient.SMB_AUTHENTICATIONS_PROPERTY;
106             public static final String SMB1_AUTHENTICATIONS = org.codelibs.fess.crawler.client.smb1.SmbClient.SMB_AUTHENTICATIONS_PROPERTY;
107             public static final String FTP_AUTHENTICATIONS = FtpClient.FTP_AUTHENTICATIONS_PROPERTY;
108             public static final String ROBOTS_TXT_ENABLED = HcHttpClient.ROBOTS_TXT_ENABLED_PROPERTY;
109             public static final String PROXY_PASSWORD = "proxyPassword";
110             public static final String PROXY_USERNAME = "proxyUsername";
111             public static final String PROXY_PORT = HcHttpClient.PROXY_PORT_PROPERTY;
112             public static final String PROXY_HOST = HcHttpClient.PROXY_HOST_PROPERTY;
113             public static final String USER_AGENT = HcHttpClient.USER_AGENT_PROPERTY;
114         }
115 
116         // xpath.*
117         public static class XPath {
118             public static final String DEFAULT_LANG = "default.lang";
119             public static final String DEFAULT_CONTENT = "default.content";
120             public static final String DEFAULT_DIGEST = "default.digest";
121             // xapth.<field>=<value>
122         }
123 
124         // config.*
125         public static class Config {
126             public static final String KEEP_ORIGINAL_BODY = "keep.original.body";
127             public static final String CLEANUP_ALL = "cleanup.all";
128             public static final String CLEANUP_URL_FILTERS = "cleanup.urlFilters";
129             public static final String JCIFS_PREFIX = "jcifs.";
130             public static final String HTML_CANONICAL_XPATH = "html.canonical.xpath";
131             public static final String HTML_PRUNED_TAGS = "html.pruned.tags";
132             public static final String PIPELINE = "pipeline";
133             public static final String IGNORE_ROBOTS_TAGS = "ignore.robots.tags";
134             public static final String SCRIPT_TYPE = "script.type";
135             public static final String HTML_CHILD_URL_RULES = "html.child.url.rules";
136             public static final String CRAWL_ORDER = "crawl.order";
137         }
138 
139         // meta.*
140         // meta.<field>=<value>
141 
142         // value.*
143         // value.<field>=<value>
144 
145         // script.*
146         // script.<field>=<value>
147 
148         // field.*
149         // field.<field>=<value>
150     }
151 }