1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.service;
17
18 import java.io.PrintWriter;
19 import java.util.ArrayList;
20 import java.util.Collections;
21 import java.util.List;
22 import java.util.regex.Pattern;
23
24 import javax.annotation.Resource;
25
26 import org.apache.commons.io.output.StringBuilderWriter;
27 import org.apache.commons.lang3.StringUtils;
28 import org.codelibs.core.beans.util.BeanUtil;
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.fess.Constants;
31 import org.codelibs.fess.app.pager.FailureUrlPager;
32 import org.codelibs.fess.es.config.cbean.FailureUrlCB;
33 import org.codelibs.fess.es.config.exbhv.FailureUrlBhv;
34 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
35 import org.codelibs.fess.es.config.exentity.FailureUrl;
36 import org.codelibs.fess.exception.ContainerNotAvailableException;
37 import org.codelibs.fess.helper.SystemHelper;
38 import org.codelibs.fess.mylasta.direction.FessConfig;
39 import org.codelibs.fess.util.ComponentUtil;
40 import org.dbflute.cbean.result.ListResultBean;
41 import org.dbflute.cbean.result.PagingResultBean;
42 import org.dbflute.optional.OptionalEntity;
43
44 public class FailureUrlService {
45
46 @Resource
47 protected FailureUrlBhv failureUrlBhv;
48
49 @Resource
50 protected FessConfig fessConfig;
51
52 public List<FailureUrl> getFailureUrlList(final FailureUrlPager failureUrlPager) {
53
54 final PagingResultBean<FailureUrl> failureUrlList = failureUrlBhv.selectPage(cb -> {
55 cb.paging(failureUrlPager.getPageSize(), failureUrlPager.getCurrentPageNumber());
56 setupListCondition(cb, failureUrlPager);
57 });
58
59
60 BeanUtil.copyBeanToBean(failureUrlList, failureUrlPager, option -> option.include(Constants.PAGER_CONVERSION_RULE));
61 failureUrlPager.setPageNumberList(failureUrlList.pageRange(op -> {
62 op.rangeSize(fessConfig.getPagingPageRangeSizeAsInteger());
63 }).createPageNumberList());
64
65 return failureUrlList;
66 }
67
68 public OptionalEntity<FailureUrl> getFailureUrl(final String id) {
69 return failureUrlBhv.selectByPK(id);
70 }
71
72 public void store(final FailureUrl failureUrl) {
73
74 failureUrlBhv.insertOrUpdate(failureUrl, op -> {
75 op.setRefreshPolicy(Constants.TRUE);
76 });
77
78 }
79
80 public void delete(final FailureUrl failureUrl) {
81
82 failureUrlBhv.delete(failureUrl, op -> {
83 op.setRefreshPolicy(Constants.TRUE);
84 });
85
86 }
87
88 protected void setupListCondition(final FailureUrlCB cb, final FailureUrlPager failureUrlPager) {
89 if (failureUrlPager.id != null) {
90 cb.query().docMeta().setId_Equal(failureUrlPager.id);
91 }
92
93
94
95 cb.query().addOrderBy_LastAccessTime_Desc();
96
97 buildSearchCondition(failureUrlPager, cb);
98 }
99
100 public void deleteAll(final FailureUrlPager failureUrlPager) {
101 failureUrlBhv.queryDelete(cb -> {
102 buildSearchCondition(failureUrlPager, cb);
103 });
104 }
105
106 private void buildSearchCondition(final FailureUrlPager failureUrlPager, final FailureUrlCB cb) {
107
108 if (StringUtil.isNotBlank(failureUrlPager.url)) {
109 cb.query().setUrl_Wildcard(failureUrlPager.url);
110 }
111
112 if (StringUtil.isNotBlank(failureUrlPager.errorCountMax)) {
113 cb.query().setErrorCount_LessEqual(Integer.parseInt(failureUrlPager.errorCountMax));
114 }
115 if (StringUtil.isNotBlank(failureUrlPager.errorCountMin)) {
116 cb.query().setErrorCount_GreaterEqual(Integer.parseInt(failureUrlPager.errorCountMin));
117 }
118
119 if (StringUtil.isNotBlank(failureUrlPager.errorName)) {
120 cb.query().setErrorName_Wildcard(failureUrlPager.errorName);
121 }
122
123 }
124
125 public List<String> getExcludedUrlList(final String configId) {
126 final int failureCount = fessConfig.getFailureCountThreshold();
127 final String ignoreFailureType = fessConfig.getIgnoreFailureType();
128
129 if (failureCount < 0) {
130 return Collections.emptyList();
131 }
132
133 final int count = failureCount;
134 final ListResultBean<FailureUrl> list = failureUrlBhv.selectList(cb -> {
135 cb.query().setConfigId_Equal(configId);
136 cb.query().setErrorCount_GreaterEqual(count);
137 cb.fetchFirst(fessConfig.getPageFailureUrlMaxFetchSizeAsInteger());
138 });
139 if (list.isEmpty()) {
140 return Collections.emptyList();
141 }
142
143 Pattern pattern = null;
144 if (StringUtil.isNotBlank(ignoreFailureType)) {
145 pattern = Pattern.compile(ignoreFailureType);
146 }
147 final List<String> urlList = new ArrayList<>();
148 for (final FailureUrl failureUrl : list) {
149 if (pattern != null) {
150 if (!pattern.matcher(failureUrl.getErrorName()).matches()) {
151 urlList.add(failureUrl.getUrl());
152 }
153 } else {
154 urlList.add(failureUrl.getUrl());
155 }
156 }
157 return urlList;
158 }
159
160 public void deleteByConfigId(final String configId) {
161 failureUrlBhv.queryDelete(cb -> {
162 cb.query().setConfigId_Equal(configId);
163 });
164 }
165
166 public void store(final CrawlingConfig crawlingConfig, final String errorName, final String url, final Throwable e) {
167 if (e instanceof ContainerNotAvailableException) {
168 return;
169 }
170
171 final FailureUrlBhv bhv = ComponentUtil.getComponent(FailureUrlBhv.class);
172 final FailureUrl failureUrl = bhv.selectEntity(cb -> {
173 cb.query().setUrl_Equal(url);
174 if (crawlingConfig != null) {
175 cb.query().setConfigId_Equal(crawlingConfig.getConfigId());
176 }
177 }).map(entity -> {
178 entity.setErrorCount(entity.getErrorCount() + 1);
179 return entity;
180 }).orElseGet(() -> {
181 final FailureUrl entity = new FailureUrl();
182 entity.setErrorCount(1);
183 entity.setUrl(url);
184 if (crawlingConfig != null) {
185 entity.setConfigId(crawlingConfig.getConfigId());
186 }
187 return entity;
188 });
189
190 failureUrl.setErrorName(errorName);
191 failureUrl.setErrorLog(StringUtils.abbreviate(getStackTrace(e), 4000));
192 failureUrl.setLastAccessTime(ComponentUtil.getSystemHelper().getCurrentTimeAsLong());
193 failureUrl.setThreadName(Thread.currentThread().getName());
194
195 bhv.insertOrUpdate(failureUrl, op -> {
196 op.setRefreshPolicy(Constants.TRUE);
197 });
198 }
199
200 private String getStackTrace(final Throwable t) {
201 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
202 final StringBuilderWriter sw = new StringBuilderWriter();
203 final PrintWriter pw = new PrintWriter(sw, true);
204 t.printStackTrace(pw);
205 return systemHelper.abbreviateLongText(sw.toString());
206 }
207 }