1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.ds;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.util.ArrayList;
21 import java.util.Date;
22 import java.util.HashMap;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.stream.Collectors;
26
27 import org.apache.logging.log4j.LogManager;
28 import org.apache.logging.log4j.Logger;
29 import org.codelibs.core.lang.StringUtil;
30 import org.codelibs.core.lang.ThreadUtil;
31 import org.codelibs.core.misc.Pair;
32 import org.codelibs.fess.Constants;
33 import org.codelibs.fess.ds.callback.IndexUpdateCallback;
34 import org.codelibs.fess.es.config.exentity.DataConfig;
35 import org.codelibs.fess.helper.CrawlingInfoHelper;
36 import org.codelibs.fess.helper.SystemHelper;
37 import org.codelibs.fess.mylasta.direction.FessConfig;
38 import org.codelibs.fess.util.ComponentUtil;
39
40 public abstract class AbstractDataStore implements DataStore {
41
42 private static final Logger logger = LogManager.getLogger(AbstractDataStore.class);
43
44 protected static final String SCRIPT_TYPE = "script_type";
45
46 public String mimeType = "application/datastore";
47
48 protected boolean alive = true;
49
50 public void register() {
51 ComponentUtil.getDataStoreFactory().add(getName(), this);
52 }
53
54 protected abstract String getName();
55
56 @Override
57 public void stop() {
58 alive = false;
59 }
60
61 @Override
62 public void store(final DataConfig config, final IndexUpdateCallback callback, final Map<String, String> initParamMap) {
63 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
64 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
65 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(config);
66 final FessConfig fessConfig = ComponentUtil.getFessConfig();
67 final Map<String, String> paramEnvMap = systemHelper.getFilteredEnvMap(fessConfig.getCrawlerDataEnvParamKeyPattern());
68 final Map<String, String> configParamMap = config.getHandlerParameterMap().entrySet().stream().map(e -> {
69 final String key = e.getKey();
70 String value = e.getValue();
71 for (final Map.Entry<String, String> entry : paramEnvMap.entrySet()) {
72 value = value.replace("${" + entry.getKey() + "}", entry.getValue());
73 }
74 return new Pair<>(key, value);
75 }).collect(Collectors.toMap(Pair<String, String>::getFirst, Pair<String, String>::getSecond));
76 final Map<String, String> configScriptMap = config.getHandlerScriptMap();
77
78 initParamMap.putAll(configParamMap);
79 final Map<String, String> paramMap = initParamMap;
80
81
82 final Map<String, Object> defaultDataMap = new HashMap<>();
83
84
85 final String configId = config.getConfigId();
86 if (configId != null) {
87 defaultDataMap.put(fessConfig.getIndexFieldConfigId(), configId);
88 }
89
90 if (documentExpires != null) {
91 defaultDataMap.put(fessConfig.getIndexFieldExpires(), documentExpires);
92 }
93
94 defaultDataMap.put(fessConfig.getIndexFieldSegment(), initParamMap.get(Constants.SESSION_ID));
95
96 defaultDataMap.put(fessConfig.getIndexFieldCreated(), systemHelper.getCurrentTime());
97
98 defaultDataMap.put(fessConfig.getIndexFieldBoost(), config.getBoost().toString());
99
100
101 final List<String> roleTypeList = new ArrayList<>();
102 stream(config.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
103 defaultDataMap.put(fessConfig.getIndexFieldRole(), roleTypeList);
104
105 defaultDataMap.put(fessConfig.getIndexFieldMimetype(), mimeType);
106
107
108
109
110
111
112
113
114
115
116
117
118 defaultDataMap.put(fessConfig.getIndexFieldVirtualHost(),
119 stream(config.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
120
121 storeData(config, callback, new ParamMap<>(paramMap), configScriptMap, defaultDataMap);
122
123 }
124
125 protected String getScriptType(final Map<String, String> paramMap) {
126 final String value = paramMap.get(SCRIPT_TYPE);
127 if (StringUtil.isBlank(value)) {
128 return Constants.DEFAULT_SCRIPT;
129 }
130 return value;
131 }
132
133 protected Object convertValue(final String scriptType, final String template, final Map<String, Object> paramMap) {
134 if (StringUtil.isEmpty(template)) {
135 return StringUtil.EMPTY;
136 }
137
138 if (paramMap.containsKey(template)) {
139 return paramMap.get(template);
140 }
141
142 return ComponentUtil.getScriptEngineFactory().getScriptEngine(scriptType).evaluate(template, paramMap);
143 }
144
145 protected long getReadInterval(final Map<String, String> paramMap) {
146 long readInterval = 0;
147 final String value = paramMap.get("readInterval");
148 if (StringUtil.isNotBlank(value)) {
149 try {
150 readInterval = Long.parseLong(value);
151 } catch (final NumberFormatException e) {
152 logger.warn("Invalid read interval: {}", value);
153 }
154 }
155 return readInterval;
156 }
157
158 protected void sleep(final long interval) {
159 ThreadUtil.sleepQuietly(interval);
160 }
161
162 protected abstract void storeData(DataConfig dataConfig, IndexUpdateCallback callback, Map<String, String> paramMap,
163 Map<String, String> scriptMap, Map<String, Object> defaultDataMap);
164 }