1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import java.util.ArrayList;
19 import java.util.Collections;
20 import java.util.List;
21 import java.util.Map;
22 import java.util.concurrent.ConcurrentHashMap;
23 import java.util.concurrent.ExecutionException;
24 import java.util.concurrent.TimeUnit;
25 import java.util.regex.Pattern;
26
27 import javax.annotation.PostConstruct;
28
29 import org.apache.logging.log4j.LogManager;
30 import org.apache.logging.log4j.Logger;
31 import org.codelibs.core.lang.StringUtil;
32 import org.codelibs.fess.Constants;
33 import org.codelibs.fess.app.service.DataConfigService;
34 import org.codelibs.fess.app.service.FileConfigService;
35 import org.codelibs.fess.app.service.WebConfigService;
36 import org.codelibs.fess.es.config.exbhv.DataConfigBhv;
37 import org.codelibs.fess.es.config.exbhv.FailureUrlBhv;
38 import org.codelibs.fess.es.config.exbhv.FileConfigBhv;
39 import org.codelibs.fess.es.config.exbhv.WebConfigBhv;
40 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
41 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
42 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigType;
43 import org.codelibs.fess.es.config.exentity.CrawlingConfig.Param.Config;
44 import org.codelibs.fess.es.config.exentity.DataConfig;
45 import org.codelibs.fess.es.config.exentity.FailureUrl;
46 import org.codelibs.fess.es.config.exentity.FileConfig;
47 import org.codelibs.fess.es.config.exentity.WebConfig;
48 import org.codelibs.fess.mylasta.direction.FessConfig;
49 import org.codelibs.fess.util.ComponentUtil;
50 import org.dbflute.cbean.result.ListResultBean;
51 import org.dbflute.optional.OptionalEntity;
52 import org.dbflute.optional.OptionalThing;
53
54 import com.google.common.cache.Cache;
55 import com.google.common.cache.CacheBuilder;
56
57 public class CrawlingConfigHelper {
58
59 private static final Logger logger = LogManager.getLogger(CrawlingConfigHelper.class);
60
61 protected final Map<String, CrawlingConfig> crawlingConfigMap = new ConcurrentHashMap<>();
62
63 protected int count = 1;
64
65 protected Cache<String, CrawlingConfig> crawlingConfigCache;
66
67 @PostConstruct
68 public void init() {
69 if (logger.isDebugEnabled()) {
70 logger.debug("Initialize {}", this.getClass().getSimpleName());
71 }
72 crawlingConfigCache = CacheBuilder.newBuilder().maximumSize(100).expireAfterWrite(10, TimeUnit.MINUTES).build();
73 }
74
75 public ConfigType getConfigType(final String configId) {
76 if (configId == null || configId.length() < 2) {
77 return null;
78 }
79 final String configType = configId.substring(0, 1);
80 if (ConfigType.WEB.getTypePrefix().equals(configType)) {
81 return ConfigType.WEB;
82 }
83 if (ConfigType.FILE.getTypePrefix().equals(configType)) {
84 return ConfigType.FILE;
85 }
86 if (ConfigType.DATA.getTypePrefix().equals(configType)) {
87 return ConfigType.DATA;
88 }
89 return null;
90 }
91
92 protected String getId(final String configId) {
93 if (configId == null || configId.length() < 2) {
94 return null;
95 }
96 return configId.substring(1);
97 }
98
99 public CrawlingConfig getCrawlingConfig(final String configId) {
100 try {
101 return crawlingConfigCache.get(configId, () -> {
102 final ConfigType configType = getConfigType(configId);
103 if (configType == null) {
104 return null;
105 }
106 final String id = getId(configId);
107 if (id == null) {
108 return null;
109 }
110 switch (configType) {
111 case WEB:
112 final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
113 return webConfigService.getWebConfig(id).get();
114 case FILE:
115 final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
116 return fileConfigService.getFileConfig(id).get();
117 case DATA:
118 final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
119 return dataConfigService.getDataConfig(id).get();
120 default:
121 return null;
122 }
123 });
124 } catch (final ExecutionException e) {
125 logger.warn("Failed to access a crawling config cache: {}", configId, e);
126 return null;
127 }
128 }
129
130 public OptionalThing<String> getPipeline(final String configId) {
131 final CrawlingConfig config = getCrawlingConfig(configId);
132 if (config == null) {
133 return OptionalThing.empty();
134 }
135 final String pipeline = config.getConfigParameterMap(ConfigName.CONFIG).get(Config.PIPELINE);
136 if (StringUtil.isBlank(pipeline)) {
137 return OptionalThing.empty();
138 }
139 return OptionalThing.of(pipeline);
140 }
141
142 public void refresh() {
143 crawlingConfigCache.invalidateAll();
144 }
145
146 public synchronized String store(final String sessionId, final CrawlingConfig crawlingConfig) {
147 final String sessionCountId = sessionId + "-" + count;
148 crawlingConfigMap.put(sessionCountId, crawlingConfig);
149 count++;
150 return sessionCountId;
151 }
152
153 public void remove(final String sessionId) {
154 crawlingConfigMap.remove(sessionId);
155 }
156
157 public CrawlingConfig get(final String sessionId) {
158 return crawlingConfigMap.get(sessionId);
159 }
160
161 public List<WebConfig> getAllWebConfigList() {
162 return getAllWebConfigList(true, true, true, null);
163 }
164
165 public List<WebConfig> getWebConfigListByIds(final List<String> idList) {
166 if (idList == null) {
167 return getAllWebConfigList();
168 }
169 return getAllWebConfigList(true, true, false, idList);
170 }
171
172 public List<WebConfig> getAllWebConfigList(final boolean withLabelType, final boolean withRoleType, final boolean available,
173 final List<String> idList) {
174 return ComponentUtil.getComponent(WebConfigBhv.class).selectList(cb -> {
175 if (available) {
176 cb.query().setAvailable_Equal(Constants.T);
177 }
178 if (idList != null) {
179 cb.query().setId_InScope(idList);
180 }
181 cb.query().setName_NotEqual(ComponentUtil.getFessConfig().getFormAdminDefaultTemplateName());
182 cb.query().addOrderBy_SortOrder_Asc();
183 cb.query().addOrderBy_Name_Asc();
184 cb.fetchFirst(ComponentUtil.getFessConfig().getPageWebConfigMaxFetchSizeAsInteger());
185 });
186 }
187
188 public List<FileConfig> getAllFileConfigList() {
189 return getAllFileConfigList(true, true, true, null);
190 }
191
192 public List<FileConfig> getFileConfigListByIds(final List<String> idList) {
193 if (idList == null) {
194 return getAllFileConfigList();
195 }
196 return getAllFileConfigList(true, true, false, idList);
197 }
198
199 public List<FileConfig> getAllFileConfigList(final boolean withLabelType, final boolean withRoleType, final boolean available,
200 final List<String> idList) {
201 return ComponentUtil.getComponent(FileConfigBhv.class).selectList(cb -> {
202 if (available) {
203 cb.query().setAvailable_Equal(Constants.T);
204 }
205 if (idList != null) {
206 cb.query().setId_InScope(idList);
207 }
208 cb.query().setName_NotEqual(ComponentUtil.getFessConfig().getFormAdminDefaultTemplateName());
209 cb.query().addOrderBy_SortOrder_Asc();
210 cb.query().addOrderBy_Name_Asc();
211 cb.fetchFirst(ComponentUtil.getFessConfig().getPageFileConfigMaxFetchSizeAsInteger());
212 });
213 }
214
215 public List<DataConfig> getAllDataConfigList() {
216 return getAllDataConfigList(true, true, true, null);
217 }
218
219 public List<DataConfig> getDataConfigListByIds(final List<String> idList) {
220 if (idList == null) {
221 return getAllDataConfigList();
222 }
223 return getAllDataConfigList(true, true, false, idList);
224 }
225
226 public List<DataConfig> getAllDataConfigList(final boolean withLabelType, final boolean withRoleType, final boolean available,
227 final List<String> idList) {
228 return ComponentUtil.getComponent(DataConfigBhv.class).selectList(cb -> {
229 if (available) {
230 cb.query().setAvailable_Equal(Constants.T);
231 }
232 if (idList != null) {
233 cb.query().setId_InScope(idList);
234 }
235 cb.query().setName_NotEqual(ComponentUtil.getFessConfig().getFormAdminDefaultTemplateName());
236 cb.query().addOrderBy_SortOrder_Asc();
237 cb.query().addOrderBy_Name_Asc();
238 cb.fetchFirst(ComponentUtil.getFessConfig().getPageDataConfigMaxFetchSizeAsInteger());
239 });
240 }
241
242 public List<String> getExcludedUrlList(final String configId) {
243 final FessConfig fessConfig = ComponentUtil.getFessConfig();
244 final int failureCount = fessConfig.getFailureCountThreshold();
245 final String ignoreFailureType = fessConfig.getIgnoreFailureType();
246
247 if (failureCount < 0) {
248 return Collections.emptyList();
249 }
250
251 final int count = failureCount;
252 final ListResultBean<FailureUrl> list = ComponentUtil.getComponent(FailureUrlBhv.class).selectList(cb -> {
253 cb.query().setConfigId_Equal(configId);
254 cb.query().setErrorCount_GreaterEqual(count);
255 cb.fetchFirst(fessConfig.getPageFailureUrlMaxFetchSizeAsInteger());
256 });
257 if (list.isEmpty()) {
258 return Collections.emptyList();
259 }
260
261 Pattern pattern = null;
262 if (StringUtil.isNotBlank(ignoreFailureType)) {
263 pattern = Pattern.compile(ignoreFailureType);
264 }
265 final List<String> urlList = new ArrayList<>();
266 for (final FailureUrl failureUrl : list) {
267 if (pattern != null) {
268 if (!pattern.matcher(failureUrl.getErrorName()).matches()) {
269 urlList.add(failureUrl.getUrl());
270 }
271 } else {
272 urlList.add(failureUrl.getUrl());
273 }
274 }
275 return urlList;
276 }
277
278 public OptionalEntity<CrawlingConfig> getDefaultConfig(final ConfigType configType) {
279 final String name = ComponentUtil.getFessConfig().getFormAdminDefaultTemplateName();
280
281 switch (configType) {
282 case WEB:
283 final WebConfigService webConfigService = ComponentUtil.getComponent(WebConfigService.class);
284 return webConfigService.getWebConfigByName(name).map(o -> (CrawlingConfig) o);
285 case FILE:
286 final FileConfigService fileConfigService = ComponentUtil.getComponent(FileConfigService.class);
287 return fileConfigService.getFileConfigByName(name).map(o -> (CrawlingConfig) o);
288 case DATA:
289 final DataConfigService dataConfigService = ComponentUtil.getComponent(DataConfigService.class);
290 return dataConfigService.getDataConfigByName(name).map(o -> (CrawlingConfig) o);
291 default:
292 return OptionalEntity.empty();
293 }
294 }
295 }