1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.ds.impl;
17
18 import java.util.Map;
19 import java.util.concurrent.atomic.AtomicLong;
20
21 import javax.annotation.PostConstruct;
22
23 import org.codelibs.fess.ds.IndexUpdateCallback;
24 import org.codelibs.fess.es.client.FessEsClient;
25 import org.codelibs.fess.exception.DataStoreException;
26 import org.codelibs.fess.helper.CrawlingInfoHelper;
27 import org.codelibs.fess.helper.IndexingHelper;
28 import org.codelibs.fess.helper.SearchLogHelper;
29 import org.codelibs.fess.helper.SystemHelper;
30 import org.codelibs.fess.mylasta.direction.FessConfig;
31 import org.codelibs.fess.util.ComponentUtil;
32 import org.codelibs.fess.util.DocList;
33 import org.codelibs.fess.util.DocumentUtil;
34 import org.slf4j.Logger;
35 import org.slf4j.LoggerFactory;
36
37 public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
38 private static final Logger logger = LoggerFactory.getLogger(IndexUpdateCallbackImpl.class);
39
40 protected AtomicLong documentSize = new AtomicLong(0);
41
42 protected volatile long executeTime = 0;
43
44 protected final DocList docList = new DocList();
45
46 protected long maxDocumentRequestSize;
47
48 @PostConstruct
49 public void init() {
50 maxDocumentRequestSize = ComponentUtil.getFessConfig().getIndexerDataMaxDocumentRequestSizeAsInteger().longValue();
51 }
52
53
54
55
56 @Override
57 public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
58 final long startTime = System.currentTimeMillis();
59 final FessConfig fessConfig = ComponentUtil.getFessConfig();
60 final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
61
62 if (logger.isDebugEnabled()) {
63 logger.debug("Adding " + dataMap);
64 }
65
66
67 final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
68 if (urlObj == null) {
69 throw new DataStoreException("url is null. dataMap=" + dataMap);
70 }
71
72 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
73 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
74 dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
75
76 final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
77
78 if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
79 addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
80 }
81
82 if (fessConfig.getIndexerFavoriteCountEnabledAsBoolean()) {
83 addFavoriteCountField(dataMap, url, fessConfig.getIndexFieldFavoriteCount());
84 }
85
86 if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
87 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
88 dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
89 }
90
91 synchronized (docList) {
92 docList.add(dataMap);
93 if (logger.isDebugEnabled()) {
94 logger.debug("Added the document. " + "The number of a document cache is " + docList.size() + ".");
95 }
96
97 final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class);
98 if (contentLength != null) {
99 docList.addContentSize(contentLength.longValue());
100 if (docList.getContentSize() >= maxDocumentRequestSize) {
101 indexingHelper.sendDocuments(fessEsClient, docList);
102 }
103 } else if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
104 indexingHelper.sendDocuments(fessEsClient, docList);
105 }
106 executeTime += System.currentTimeMillis() - startTime;
107 }
108
109 documentSize.getAndIncrement();
110
111 if (logger.isDebugEnabled()) {
112 logger.debug("The number of an added document is " + documentSize.get() + ".");
113 }
114
115 }
116
117 @Override
118 public void commit() {
119 synchronized (docList) {
120 if (!docList.isEmpty()) {
121 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
122 final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
123 indexingHelper.sendDocuments(fessEsClient, docList);
124 }
125 }
126 }
127
128 protected void addClickCountField(final Map<String, Object> doc, final String url, final String clickCountField) {
129 final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
130 final int count = searchLogHelper.getClickCount(url);
131 doc.put(clickCountField, count);
132 if (logger.isDebugEnabled()) {
133 logger.debug("Click Count: " + count + ", url: " + url);
134 }
135 }
136
137 protected void addFavoriteCountField(final Map<String, Object> doc, final String url, final String favoriteCountField) {
138 final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
139 final long count = searchLogHelper.getFavoriteCount(url);
140 doc.put(favoriteCountField, count);
141 if (logger.isDebugEnabled()) {
142 logger.debug("Favorite Count: " + count + ", url: " + url);
143 }
144 }
145
146 @Override
147 public long getDocumentSize() {
148 return documentSize.get();
149 }
150
151 @Override
152 public long getExecuteTime() {
153 return executeTime;
154 }
155
156 }