1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.app.service;
17
18 import java.io.IOException;
19 import java.io.Reader;
20 import java.io.Writer;
21 import java.text.DateFormat;
22 import java.text.SimpleDateFormat;
23 import java.time.LocalDateTime;
24 import java.time.format.DateTimeFormatter;
25 import java.util.ArrayList;
26 import java.util.Collections;
27 import java.util.List;
28 import java.util.Set;
29 import java.util.stream.Collectors;
30
31 import javax.annotation.Resource;
32
33 import org.apache.logging.log4j.LogManager;
34 import org.apache.logging.log4j.Logger;
35 import org.codelibs.core.CoreLibConstants;
36 import org.codelibs.core.beans.util.BeanUtil;
37 import org.codelibs.core.lang.StringUtil;
38 import org.codelibs.fess.Constants;
39 import org.codelibs.fess.app.pager.CrawlingInfoPager;
40 import org.codelibs.fess.es.config.cbean.CrawlingInfoCB;
41 import org.codelibs.fess.es.config.exbhv.CrawlingInfoBhv;
42 import org.codelibs.fess.es.config.exbhv.CrawlingInfoParamBhv;
43 import org.codelibs.fess.es.config.exentity.CrawlingInfo;
44 import org.codelibs.fess.es.config.exentity.CrawlingInfoParam;
45 import org.codelibs.fess.exception.FessSystemException;
46 import org.codelibs.fess.mylasta.direction.FessConfig;
47 import org.codelibs.fess.util.ComponentUtil;
48 import org.dbflute.bhv.readable.EntityRowHandler;
49 import org.dbflute.cbean.result.ListResultBean;
50 import org.dbflute.cbean.result.PagingResultBean;
51 import org.dbflute.optional.OptionalEntity;
52
53 import com.orangesignal.csv.CsvConfig;
54 import com.orangesignal.csv.CsvReader;
55 import com.orangesignal.csv.CsvWriter;
56
57 public class CrawlingInfoService {
58
59 private static final Logger logger = LogManager.getLogger(CrawlingInfoService.class);
60
61 @Resource
62 protected CrawlingInfoParamBhv crawlingInfoParamBhv;
63
64 @Resource
65 protected CrawlingInfoBhv crawlingInfoBhv;
66
67 @Resource
68 protected FessConfig fessConfig;
69
70 public List<CrawlingInfo> getCrawlingInfoList(final CrawlingInfoPager crawlingInfoPager) {
71
72 final PagingResultBean<CrawlingInfo> crawlingInfoList = crawlingInfoBhv.selectPage(cb -> {
73 cb.paging(crawlingInfoPager.getPageSize(), crawlingInfoPager.getCurrentPageNumber());
74 setupListCondition(cb, crawlingInfoPager);
75 });
76
77
78 BeanUtil.copyBeanToBean(crawlingInfoList, crawlingInfoPager, option -> option.include(Constants.PAGER_CONVERSION_RULE));
79 crawlingInfoPager.setPageNumberList(
80 crawlingInfoList.pageRange(op -> op.rangeSize(fessConfig.getPagingPageRangeSizeAsInteger())).createPageNumberList());
81
82 return crawlingInfoList;
83 }
84
85 public OptionalEntity<CrawlingInfo> getCrawlingInfo(final String id) {
86 return crawlingInfoBhv.selectByPK(id);
87 }
88
89 public void store(final CrawlingInfo crawlingInfo) {
90 setupStoreCondition(crawlingInfo);
91
92 crawlingInfoBhv.insertOrUpdate(crawlingInfo, op -> op.setRefreshPolicy(Constants.TRUE));
93
94 }
95
96 public void delete(final CrawlingInfo crawlingInfo) {
97 setupDeleteCondition(crawlingInfo);
98
99 crawlingInfoBhv.delete(crawlingInfo, op -> op.setRefreshPolicy(Constants.TRUE));
100
101 }
102
103 protected void setupListCondition(final CrawlingInfoCB cb, final CrawlingInfoPager crawlingInfoPager) {
104 if (crawlingInfoPager.id != null) {
105 cb.query().docMeta().setId_Equal(crawlingInfoPager.id);
106 }
107
108 if (StringUtil.isNotBlank(crawlingInfoPager.sessionId)) {
109 cb.query().setSessionId_Match(crawlingInfoPager.sessionId);
110 }
111 cb.query().addOrderBy_CreatedTime_Desc();
112 }
113
114 protected void setupStoreCondition(final CrawlingInfo crawlingInfo) {
115 if (crawlingInfo == null) {
116 throw new FessSystemException("Crawling Session is null.");
117 }
118 final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
119 if (crawlingInfo.getCreatedTime() == null) {
120 crawlingInfo.setCreatedTime(now);
121 }
122 }
123
124 protected void setupDeleteCondition(final CrawlingInfo crawlingInfo) {
125 crawlingInfoParamBhv.queryDelete(cb -> cb.query().setCrawlingInfoId_Equal(crawlingInfo.getId()));
126 }
127
128 public void deleteSessionIdsBefore(final String activeSessionId, final String name, final long date) {
129 final List<CrawlingInfo> crawlingInfoList = crawlingInfoBhv.selectList(cb -> {
130 cb.query().filtered((cq, cf) -> {
131 cq.setExpiredTime_LessEqual(date);
132 if (StringUtil.isNotBlank(name)) {
133 cf.setName_Equal(name);
134 }
135 if (activeSessionId != null) {
136 cf.setSessionId_NotEqual(activeSessionId);
137 }
138
139 });
140
141 cb.fetchFirst(fessConfig.getPageCrawlingInfoMaxFetchSizeAsInteger());
142 cb.specify().columnId();
143 });
144 if (!crawlingInfoList.isEmpty()) {
145 final List<String> crawlingInfoIdList = new ArrayList<>();
146 for (final CrawlingInfo cs : crawlingInfoList) {
147 crawlingInfoIdList.add(cs.getId());
148 }
149 crawlingInfoParamBhv.queryDelete(cb2 -> cb2.query().setCrawlingInfoId_InScope(crawlingInfoIdList));
150 crawlingInfoBhv.batchDelete(crawlingInfoList, op -> op.setRefreshPolicy(Constants.TRUE));
151 }
152 }
153
154 public void storeInfo(final List<CrawlingInfoParam> crawlingInfoParamList) {
155 if (crawlingInfoParamList == null) {
156 throw new FessSystemException("Crawling Session Info is null.");
157 }
158
159 final long now = ComponentUtil.getSystemHelper().getCurrentTimeAsLong();
160 for (final CrawlingInfoParam crawlingInfoParam : crawlingInfoParamList) {
161 if (crawlingInfoParam.getCreatedTime() == null) {
162 crawlingInfoParam.setCreatedTime(now);
163 }
164 }
165 crawlingInfoParamBhv.batchInsert(crawlingInfoParamList, op -> op.setRefreshPolicy(Constants.TRUE));
166 }
167
168 public List<CrawlingInfoParam> getCrawlingInfoParamList(final String id) {
169 return crawlingInfoParamBhv.selectList(cb -> {
170 cb.query().setCrawlingInfoId_Equal(id);
171 cb.query().addOrderBy_CreatedTime_Asc();
172 cb.fetchFirst(fessConfig.getPageCrawlingInfoParamMaxFetchSizeAsInteger());
173 });
174 }
175
176 public List<CrawlingInfoParam> getLastCrawlingInfoParamList(final String sessionId) {
177 final CrawlingInfo crawlingInfo = getLast(sessionId);
178 if (crawlingInfo == null) {
179 return Collections.emptyList();
180 }
181 final FessConfig fessConfig = ComponentUtil.getFessConfig();
182 return crawlingInfoParamBhv.selectList(cb -> {
183 cb.query().setCrawlingInfoId_Equal(crawlingInfo.getId());
184 cb.query().addOrderBy_CreatedTime_Asc();
185 cb.paging(fessConfig.getPageCrawlingInfoParamMaxFetchSizeAsInteger(), 1);
186 });
187 }
188
189 public void deleteOldSessions(final Set<String> activeSessionId) {
190 final List<CrawlingInfo> activeSessionList =
191 activeSessionId.isEmpty() ? Collections.emptyList() : crawlingInfoBhv.selectList(cb -> {
192 cb.query().setSessionId_InScope(activeSessionId);
193 cb.fetchFirst(fessConfig.getPageCrawlingInfoMaxFetchSizeAsInteger());
194 cb.specify().columnId();
195 });
196 final List<String> idList = activeSessionList.stream().map(CrawlingInfo::getId).collect(Collectors.toList());
197 crawlingInfoParamBhv.queryDelete(cb1 -> cb1.query().filtered((cq, cf) -> {
198 cq.matchAll();
199 if (!idList.isEmpty()) {
200 cf.not(subCf -> subCf.setCrawlingInfoId_InScope(idList));
201 }
202 }));
203 crawlingInfoBhv.queryDelete(cb2 -> cb2.query().filtered((cq, cf) -> {
204 cq.matchAll();
205 if (!idList.isEmpty()) {
206 cf.not(subCf -> subCf.setId_InScope(idList));
207 }
208 }));
209 }
210
211 public void importCsv(final Reader reader) {
212 @SuppressWarnings("resource")
213 final CsvReader csvReader = new CsvReader(reader, new CsvConfig());
214 final DateFormat formatter = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
215 try {
216 List<String> list;
217 csvReader.readValues();
218 while ((list = csvReader.readValues()) != null) {
219 try {
220 final String sessionId = list.get(0);
221 CrawlingInfo crawlingInfo = crawlingInfoBhv.selectEntity(cb -> {
222 cb.query().setSessionId_Equal(sessionId);
223 cb.specify().columnSessionId();
224 }).orElse(null);
225 if (crawlingInfo == null) {
226 crawlingInfo = new CrawlingInfo();
227 crawlingInfo.setSessionId(list.get(0));
228 crawlingInfo.setCreatedTime(formatter.parse(list.get(1)).getTime());
229 crawlingInfoBhv.insert(crawlingInfo, op -> op.setRefreshPolicy(Constants.TRUE));
230 }
231
232 final CrawlingInfoParam entity = new CrawlingInfoParam();
233 entity.setCrawlingInfoId(crawlingInfo.getId());
234 entity.setKey(list.get(2));
235 entity.setValue(list.get(3));
236 entity.setCreatedTime(formatter.parse(list.get(4)).getTime());
237 crawlingInfoParamBhv.insert(entity, op -> op.setRefreshPolicy(Constants.TRUE));
238 } catch (final Exception e) {
239 logger.warn("Failed to read a click log: {}", list, e);
240 }
241 }
242 } catch (final IOException e) {
243 logger.warn("Failed to read a click log.", e);
244 }
245 }
246
247 public void exportCsv(final Writer writer) {
248 final CsvConfig cfg = new CsvConfig(',', '"', '"');
249 cfg.setEscapeDisabled(false);
250 cfg.setQuoteDisabled(false);
251 @SuppressWarnings("resource")
252 final CsvWriter csvWriter = new CsvWriter(writer, cfg);
253 try {
254 final List<String> list = new ArrayList<>();
255 list.add("SessionId");
256 list.add("SessionCreatedTime");
257 list.add("Key");
258 list.add("Value");
259 list.add("CreatedTime");
260 csvWriter.writeValues(list);
261 final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
262 crawlingInfoParamBhv.selectCursor(cb -> cb.query().matchAll(), new EntityRowHandler<CrawlingInfoParam>() {
263 @Override
264 public void handle(final CrawlingInfoParam entity) {
265 final List<String> list = new ArrayList<>();
266 entity.getCrawlingInfo().ifPresent(crawlingInfo -> {
267 addToList(list, crawlingInfo.getSessionId());
268 addToList(list, crawlingInfo.getCreatedTime());
269 });
270
271 if (!entity.getCrawlingInfo().isPresent()) {
272 addToList(list, "");
273 addToList(list, "");
274 }
275 addToList(list, entity.getKey());
276 addToList(list, entity.getValue());
277 addToList(list, entity.getCreatedTime());
278 try {
279 csvWriter.writeValues(list);
280 } catch (final IOException e) {
281 logger.warn("Failed to write a crawling session info: {}", entity, e);
282 }
283 }
284
285 private void addToList(final List<String> list, final Object value) {
286 if (value == null) {
287 list.add(StringUtil.EMPTY);
288 } else if (value instanceof LocalDateTime) {
289 list.add(((LocalDateTime) value).format(formatter));
290 } else {
291 list.add(value.toString());
292 }
293 }
294 });
295 csvWriter.flush();
296 } catch (final IOException e) {
297 logger.warn("Failed to write a crawling session info.", e);
298 }
299 }
300
301 public void deleteBefore(final long date) {
302 crawlingInfoBhv.selectBulk(cb -> cb.query().setExpiredTime_LessThan(date), list -> {
303 final List<String> idList = list.stream().map(CrawlingInfo::getId).collect(Collectors.toList());
304 crawlingInfoParamBhv.queryDelete(cb1 -> cb1.query().setCrawlingInfoId_InScope(idList));
305 crawlingInfoBhv.queryDelete(cb2 -> cb2.query().setExpiredTime_LessThan(date));
306 });
307 }
308
309 public CrawlingInfo getLast(final String sessionId) {
310 final ListResultBean<CrawlingInfo> list = crawlingInfoBhv.selectList(cb -> {
311 cb.query().setSessionId_Equal(sessionId);
312 cb.query().addOrderBy_CreatedTime_Desc();
313 cb.fetchFirst(1);
314 });
315 if (list.isEmpty()) {
316 return null;
317 }
318 return list.get(0);
319 }
320
321 }