View Javadoc
1   /*
2    * Copyright 2012-2021 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.ds;
17  
18  import static org.codelibs.core.stream.StreamUtil.stream;
19  
20  import java.util.ArrayList;
21  import java.util.Date;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.stream.Collectors;
26  
27  import org.apache.logging.log4j.LogManager;
28  import org.apache.logging.log4j.Logger;
29  import org.codelibs.core.lang.StringUtil;
30  import org.codelibs.core.lang.ThreadUtil;
31  import org.codelibs.core.misc.Pair;
32  import org.codelibs.fess.Constants;
33  import org.codelibs.fess.ds.callback.IndexUpdateCallback;
34  import org.codelibs.fess.es.config.exentity.DataConfig;
35  import org.codelibs.fess.helper.CrawlingInfoHelper;
36  import org.codelibs.fess.helper.SystemHelper;
37  import org.codelibs.fess.mylasta.direction.FessConfig;
38  import org.codelibs.fess.util.ComponentUtil;
39  
40  public abstract class AbstractDataStore implements DataStore {
41  
42      private static final Logger logger = LogManager.getLogger(AbstractDataStore.class);
43  
44      protected static final String SCRIPT_TYPE = "script_type";
45  
46      public String mimeType = "application/datastore";
47  
48      protected boolean alive = true;
49  
50      public void register() {
51          ComponentUtil.getDataStoreFactory().add(getName(), this);
52      }
53  
54      protected abstract String getName();
55  
56      @Override
57      public void stop() {
58          alive = false;
59      }
60  
61      @Override
62      public void store(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> initParamMap) {
63          final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
64          final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
65          final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
66          final FessConfig fessConfig = ComponentUtil.getFessConfig();
67          final Map<String, String> paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern());
68          final Map<String, String> configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> {
69              final String key = e.getKey();
70              String value = e.getValue();
71              for (final Map.Entry<String, String> entry : paramEnvMap.entrySet()) {
72                  value = value.replace("${" + entry.getKey() + "}", entry.getValue());
73              }
74              return new Pair<>(key, value);
75          }).collect(Collectors.toMap(Pair<String, String>::getFirst, Pair<String, String>::getSecond));
76          final Map<String, String> configScriptMap = config.getHandlerScriptMap();
77  
78          initParamMap.putAll(configParamMap);
79          final Map<String, String> paramMap = initParamMap;
80  
81          // default values
82          final Map<String, Object> defaultDataMap = new HashMap<>();
83  
84          // cid
85          final String configId = config.getConfigId();
86          if (configId != null) {
87              defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
88          }
89          //  expires
90          if (documentExpires != null) {
91              defaultDataMap.put(fessConfig.getIndexFieldExpires(), documentExpires);
92          }
93          // segment
94          defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.get(Constants.SESSION_ID));
95          // created
96          defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
97          // boost
98          defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
99          // label: labelType
100         // role: roleType
101         final List<String> roleTypeList = new ArrayList<>();
102         stream(config.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
103         defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
104         // mimetype
105         defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
106         // title
107         // content
108         // cache
109         // digest
110         // host
111         // site
112         // url
113         // anchor
114         // content_length
115         // last_modified
116         // id
117         // virtual_host
118         defaultDataMap.put(fessConfig.getIndexFieldVirtualHost(),
119                 stream(config.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
120 
121         storeData(config, callback, new ParamMap<>(paramMap), configScriptMap, defaultDataMap);
122 
123     }
124 
125     protected String getScriptType(final Map<String, String> paramMap) {
126         final String value = paramMap.get(SCRIPT_TYPE);
127         if (StringUtil.isBlank(value)) {
128             return Constants.DEFAULT_SCRIPT;
129         }
130         return value;
131     }
132 
133     protected Object convertValue(final String scriptType, final String template, final Map<String, Object> paramMap) {
134         if (StringUtil.isEmpty(template)) {
135             return StringUtil.EMPTY;
136         }
137 
138         if (paramMap.containsKey(template)) {
139             return paramMap.get(template);
140         }
141 
142         return ComponentUtil.getScriptEngineFactory().getScriptEngine(scriptType).evaluate(template, paramMap);
143     }
144 
145     protected long getReadInterval(final Map<String, String> paramMap) {
146         long readInterval = 0;
147         final String value = paramMap.get("readInterval");
148         if (StringUtil.isNotBlank(value)) {
149             try {
150                 readInterval = Long.parseLong(value);
151             } catch (final NumberFormatException e) {
152                 logger.warn("Invalid read interval: {}", value);
153             }
154         }
155         return readInterval;
156     }
157 
158     protected void sleep(final long interval) {
159         ThreadUtil.sleepQuietly(interval);
160     }
161 
162     protected abstract void storeData(DataConfig dataConfig, IndexUpdateCallback callback, Map<String, String> paramMap,
163             Map<String, String> scriptMap, Map<String, Object> defaultDataMap);
164 }