View Javadoc
1   /*
2    * Copyright 2012-2020 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.ds.callback;
17  
18  import java.util.HashSet;
19  import java.util.Map;
20  import java.util.Set;
21  import java.util.concurrent.atomic.AtomicLong;
22  
23  import javax.annotation.PostConstruct;
24  
25  import org.apache.logging.log4j.LogManager;
26  import org.apache.logging.log4j.Logger;
27  import org.codelibs.core.stream.StreamUtil;
28  import org.codelibs.fess.es.client.FessEsClient;
29  import org.codelibs.fess.exception.DataStoreException;
30  import org.codelibs.fess.helper.CrawlingInfoHelper;
31  import org.codelibs.fess.helper.IndexingHelper;
32  import org.codelibs.fess.helper.SearchLogHelper;
33  import org.codelibs.fess.helper.SystemHelper;
34  import org.codelibs.fess.mylasta.direction.FessConfig;
35  import org.codelibs.fess.util.ComponentUtil;
36  import org.codelibs.fess.util.DocList;
37  import org.codelibs.fess.util.DocumentUtil;
38  
39  public class IndexUpdateCallbackImpl implements IndexUpdateCallback {
40      private static final Logger logger = LogManager.getLogger(IndexUpdateCallbackImpl.class);
41  
42      protected AtomicLong documentSize = new AtomicLong(0);
43  
44      protected volatile long executeTime = 0;
45  
46      protected final DocListist.html#DocList">DocList docList = new DocList();
47  
48      protected long maxDocumentRequestSize;
49  
50      @PostConstruct
51      public void init() {
52          if (logger.isDebugEnabled()) {
53              logger.debug("Initialize {}", this.getClass().getSimpleName());
54          }
55          maxDocumentRequestSize = ComponentUtil.getFessConfig().getIndexerDataMaxDocumentRequestSizeAsInteger().longValue();
56      }
57  
58      /* (non-Javadoc)
59       * @see org.codelibs.fess.ds.callback.IndexUpdateCallback#store(java.util.Map)
60       */
61      @Override
62      public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
63          final long startTime = System.currentTimeMillis();
64          final FessConfig fessConfig = ComponentUtil.getFessConfig();
65          final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
66  
67          if (logger.isDebugEnabled()) {
68              logger.debug("Adding {}", dataMap);
69          }
70  
71          //   required check
72          final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
73          if (urlObj == null) {
74              throw new DataStoreException("url is null. dataMap=" + dataMap);
75          }
76  
77          final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
78          final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
79          dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
80  
81          final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
82  
83          if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
84              addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
85          }
86  
87          if (fessConfig.getIndexerFavoriteCountEnabledAsBoolean()) {
88              addFavoriteCountField(dataMap, url, fessConfig.getIndexFieldFavoriteCount());
89          }
90  
91          final Set<String> matchedLabelSet = ComponentUtil.getLabelTypeHelper().getMatchedLabelValueSet(url);
92          if (!matchedLabelSet.isEmpty()) {
93              final Set<String> newLabelSet = new HashSet<>();
94              final String[] oldLabels = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldLabel(), String[].class);
95              StreamUtil.stream(oldLabels).of(stream -> stream.forEach(newLabelSet::add));
96              matchedLabelSet.stream().forEach(newLabelSet::add);
97              dataMap.put(fessConfig.getIndexFieldLabel(), newLabelSet.toArray(new String[newLabelSet.size()]));
98          }
99  
100         if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
101             final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
102             dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
103         }
104 
105         ComponentUtil.getLanguageHelper().updateDocument(dataMap);
106 
107         synchronized (docList) {
108             docList.add(dataMap);
109             if (logger.isDebugEnabled()) {
110                 logger.debug("Added the document. The number of a document cache is {}.", docList.size());
111             }
112 
113             final Long contentLength = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldContentLength(), Long.class);
114             if (contentLength != null) {
115                 docList.addContentSize(contentLength.longValue());
116                 if (docList.getContentSize() >= maxDocumentRequestSize) {
117                     indexingHelper.sendDocuments(fessEsClient, docList);
118                 }
119             } else if (docList.size() >= fessConfig.getIndexerDataMaxDocumentCacheSizeAsInteger().intValue()) {
120                 indexingHelper.sendDocuments(fessEsClient, docList);
121             }
122             executeTime += System.currentTimeMillis() - startTime;
123         }
124 
125         documentSize.getAndIncrement();
126 
127         if (logger.isDebugEnabled()) {
128             logger.debug("The number of an added document is {}.", documentSize.get());
129         }
130 
131     }
132 
133     @Override
134     public void commit() {
135         synchronized (docList) {
136             if (!docList.isEmpty()) {
137                 final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
138                 final FessEsClient fessEsClient = ComponentUtil.getFessEsClient();
139                 indexingHelper.sendDocuments(fessEsClient, docList);
140             }
141         }
142     }
143 
144     protected void addClickCountField(final Map<String, Object> doc, final String url, final String clickCountField) {
145         final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
146         final int count = searchLogHelper.getClickCount(url);
147         doc.put(clickCountField, count);
148         if (logger.isDebugEnabled()) {
149             logger.debug("Click Count: {}, url: {}", count, url);
150         }
151     }
152 
153     protected void addFavoriteCountField(final Map<String, Object> doc, final String url, final String favoriteCountField) {
154         final SearchLogHelper searchLogHelper = ComponentUtil.getSearchLogHelper();
155         final long count = searchLogHelper.getFavoriteCount(url);
156         doc.put(favoriteCountField, count);
157         if (logger.isDebugEnabled()) {
158             logger.debug("Favorite Count: {}, url: {}", count, url);
159         }
160     }
161 
162     @Override
163     public long getDocumentSize() {
164         return documentSize.get();
165     }
166 
167     @Override
168     public long getExecuteTime() {
169         return executeTime;
170     }
171 
172 }