View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.ds;
17  
18  import static org.codelibs.core.stream.StreamUtil.stream;
19  
20  import java.util.ArrayList;
21  import java.util.Date;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.stream.Collectors;
26  
27  import org.apache.logging.log4j.LogManager;
28  import org.apache.logging.log4j.Logger;
29  import org.codelibs.core.lang.StringUtil;
30  import org.codelibs.core.lang.ThreadUtil;
31  import org.codelibs.core.misc.Pair;
32  import org.codelibs.fess.Constants;
33  import org.codelibs.fess.ds.callback.IndexUpdateCallback;
34  import org.codelibs.fess.entity.DataStoreParams;
35  import org.codelibs.fess.helper.CrawlingInfoHelper;
36  import org.codelibs.fess.helper.SystemHelper;
37  import org.codelibs.fess.mylasta.direction.FessConfig;
38  import org.codelibs.fess.opensearch.config.exentity.DataConfig;
39  import org.codelibs.fess.util.ComponentUtil;
40  
41  /**
42   * The abstract class for DataStore.
43   */
44  public abstract class AbstractDataStore implements DataStore {
45  
46      private static final Logger logger = LogManager.getLogger(AbstractDataStore.class);
47  
48      /**
49       * The script type.
50       */
51      protected static final String SCRIPT_TYPE = "script_type";
52  
53      /**
54       * The mime type.
55       */
56      public String mimeType = "application/datastore";
57  
58      /**
59       * The flag to check if the data store is alive.
60       * Volatile to ensure visibility across threads.
61       */
62      protected volatile boolean alive = true;
63  
64      /**
65       * Default constructor.
66       */
67      public AbstractDataStore() {
68          // nothing
69      }
70  
71      /**
72       * Register this data store.
73       */
74      public void register() {
75          ComponentUtil.getDataStoreFactory().add(getName(), this);
76      }
77  
78      /**
79       * Get the name of this data store.
80       * @return The name of this data store.
81       */
82      protected abstract String getName();
83  
84      @Override
85      public void stop() {
86          alive = false;
87      }
88  
89      @Override
90      public void store(final DataConfig config, final IndexUpdateCallback callback, final DataStoreParams initParamMap) {
91          final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
92          final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
93          final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
94          final FessConfig fessConfig = ComponentUtil.getFessConfig();
95          final Map<String, String> paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern());
96          final Map<String, String> configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> {
97              final String key = e.getKey();
98              String value = e.getValue();
99              for (final Map.Entry<String, String> entry : paramEnvMap.entrySet()) {
100                 value = value.replace("${" + entry.getKey() + "}", entry.getValue());
101             }
102             return new Pair<>(key, value);
103         }).collect(Collectors.toMap(Pair<String, String>::getFirst, Pair<String, String>::getSecond));
104         final Map<String, String> configScriptMap = config.getHandlerScriptMap();
105 
106         initParamMap.putAll(configParamMap);
107         final DataStoreParams paramMap = initParamMap;
108 
109         // default values
110         final Map<String, Object> defaultDataMap = new HashMap<>();
111 
112         // cid
113         final String configId = config.getConfigId();
114         if (configId != null) {
115             defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
116         }
117         //  expires
118         if (documentExpires != null) {
119             defaultDataMap.put(fessConfig.getIndexFieldExpires(), documentExpires);
120         }
121         // segment
122         defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.getAsString(Constants.SESSION_ID));
123         // created
124         defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
125         // boost
126         defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
127         // label: labelType
128         // role: roleType
129         final List<String> roleTypeList = new ArrayList<>();
130         stream(config.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
131         defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
132         // mimetype
133         defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
134         // title
135         // content
136         // cache
137         // digest
138         // host
139         // site
140         // url
141         // anchor
142         // content_length
143         // last_modified
144         // id
145         // virtual_host
146         defaultDataMap.put(fessConfig.getIndexFieldVirtualHost(),
147                 stream(config.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
148 
149         storeData(config, callback, paramMap.newInstance(), configScriptMap, defaultDataMap);
150 
151     }
152 
153     /**
154      * Get the script type.
155      * @param paramMap The parameters.
156      * @return The script type.
157      */
158     protected String getScriptType(final DataStoreParams paramMap) {
159         final String value = paramMap.getAsString(SCRIPT_TYPE);
160         if (StringUtil.isBlank(value)) {
161             return Constants.DEFAULT_SCRIPT;
162         }
163         return value;
164     }
165 
166     /**
167      * Convert the value.
168      * @param scriptType The script type.
169      * @param template The template.
170      * @param paramMap The parameters.
171      * @return The converted value.
172      */
173     protected Object convertValue(final String scriptType, final String template, final Map<String, Object> paramMap) {
174         if (StringUtil.isEmpty(template)) {
175             return StringUtil.EMPTY;
176         }
177 
178         if (paramMap.containsKey(template)) {
179             return paramMap.get(template);
180         }
181 
182         return ComponentUtil.getScriptEngineFactory().getScriptEngine(scriptType).evaluate(template, paramMap);
183     }
184 
185     /**
186      * Get the read interval.
187      * @param paramMap The parameters.
188      * @return The read interval.
189      */
190     protected long getReadInterval(final DataStoreParams paramMap) {
191         long readInterval = 0;
192         final String value = paramMap.getAsString("readInterval");
193         if (StringUtil.isNotBlank(value)) {
194             try {
195                 readInterval = Long.parseLong(value);
196             } catch (final NumberFormatException e) {
197                 logger.warn("Invalid readInterval value: '{}'. Expected: numeric value in milliseconds.", value);
198             }
199         }
200         return readInterval;
201     }
202 
203     /**
204      * Sleep for the specified interval.
205      * @param interval The interval.
206      */
207     protected void sleep(final long interval) {
208         ThreadUtil.sleepQuietly(interval);
209     }
210 
211     /**
212      * Store the data.
213      * @param dataConfig The data configuration.
214      * @param callback The callback.
215      * @param paramMap The parameters.
216      * @param scriptMap The script map.
217      * @param defaultDataMap The default data map.
218      */
219     protected abstract void storeData(DataConfig dataConfig, IndexUpdateCallback callback, DataStoreParams paramMap,
220             Map<String, String> scriptMap, Map<String, Object> defaultDataMap);
221 }