1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.ds;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.util.ArrayList;
21 import java.util.Date;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.stream.Collectors;
26
27 import org.apache.logging.log4j.LogManager;
28 import org.apache.logging.log4j.Logger;
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.core.lang.ThreadUtil;
31 import org.codelibs.core.misc.Pair;
32 import org.codelibs.fess.Constants;
33 import org.codelibs.fess.ds.callback.IndexUpdateCallback;
34 import org.codelibs.fess.entity.DataStoreParams;
35 import org.codelibs.fess.helper.CrawlingInfoHelper;
36 import org.codelibs.fess.helper.SystemHelper;
37 import org.codelibs.fess.mylasta.direction.FessConfig;
38 import org.codelibs.fess.opensearch.config.exentity.DataConfig;
39 import org.codelibs.fess.util.ComponentUtil;
40
41
42
43
44 public abstract class AbstractDataStore implements DataStore {
45
46 private static final Logger logger = LogManager.getLogger(AbstractDataStore.class);
47
48
49
50
51 protected static final String SCRIPT_TYPE = "script_type";
52
53
54
55
56 public String mimeType = "application/datastore";
57
58
59
60
61
62 protected volatile boolean alive = true;
63
64
65
66
67 public AbstractDataStore() {
68
69 }
70
71
72
73
74 public void register() {
75 ComponentUtil.getDataStoreFactory().add(getName(), this);
76 }
77
78
79
80
81
82 protected abstract String getName();
83
84 @Override
85 public void stop() {
86 alive = false;
87 }
88
89 @Override
90 public void store(final DataConfig config, final IndexUpdateCallback callback, final DataStoreParams initParamMap) {
91 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
92 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
93 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
94 final FessConfig fessConfig = ComponentUtil.getFessConfig();
95 final Map<String, String> paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern());
96 final Map<String, String> configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> {
97 final String key = e.getKey();
98 String value = e.getValue();
99 for (final Map.Entry<String, String> entry : paramEnvMap.entrySet()) {
100 value = value.replace("${" + entry.getKey() + "}", entry.getValue());
101 }
102 return new Pair<>(key, value);
103 }).collect(Collectors.toMap(Pair<String, String>::getFirst, Pair<String, String>::getSecond));
104 final Map<String, String> configScriptMap = config.getHandlerScriptMap();
105
106 initParamMap.putAll(configParamMap);
107 final DataStoreParams paramMap = initParamMap;
108
109
110 final Map<String, Object> defaultDataMap = new HashMap<>();
111
112
113 final String configId = config.getConfigId();
114 if (configId != null) {
115 defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
116 }
117
118 if (documentExpires != null) {
119 defaultDataMap.put(fessConfig.getIndexFieldExpires(), documentExpires);
120 }
121
122 defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.getAsString(Constants.SESSION_ID));
123
124 defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
125
126 defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
127
128
129 final List<String> roleTypeList = new ArrayList<>();
130 stream(config.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
131 defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
132
133 defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
134
135
136
137
138
139
140
141
142
143
144
145
146 defaultDataMap.put(fessConfig.getIndexFieldVirtualHost(),
147 stream(config.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
148
149 storeData(config, callback, paramMap.newInstance(), configScriptMap, defaultDataMap);
150
151 }
152
153
154
155
156
157
158 protected String getScriptType(final DataStoreParams paramMap) {
159 final String value = paramMap.getAsString(SCRIPT_TYPE);
160 if (StringUtil.isBlank(value)) {
161 return Constants.DEFAULT_SCRIPT;
162 }
163 return value;
164 }
165
166
167
168
169
170
171
172
173 protected Object convertValue(final String scriptType, final String template, final Map<String, Object> paramMap) {
174 if (StringUtil.isEmpty(template)) {
175 return StringUtil.EMPTY;
176 }
177
178 if (paramMap.containsKey(template)) {
179 return paramMap.get(template);
180 }
181
182 return ComponentUtil.getScriptEngineFactory().getScriptEngine(scriptType).evaluate(template, paramMap);
183 }
184
185
186
187
188
189
190 protected long getReadInterval(final DataStoreParams paramMap) {
191 long readInterval = 0;
192 final String value = paramMap.getAsString("readInterval");
193 if (StringUtil.isNotBlank(value)) {
194 try {
195 readInterval = Long.parseLong(value);
196 } catch (final NumberFormatException e) {
197 logger.warn("Invalid readInterval value: '{}'. Expected: numeric value in milliseconds.", value);
198 }
199 }
200 return readInterval;
201 }
202
203
204
205
206
207 protected void sleep(final long interval) {
208 ThreadUtil.sleepQuietly(interval);
209 }
210
211
212
213
214
215
216
217
218
219 protected abstract void storeData(DataConfig dataConfig, IndexUpdateCallback callback, DataStoreParams paramMap,
220 Map<String, String> scriptMap, Map<String, Object> defaultDataMap);
221 }