1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import java.net.URLDecoder;
19 import java.util.Arrays;
20 import java.util.Collections;
21 import java.util.HashMap;
22 import java.util.Map;
23
24 import org.apache.commons.lang3.StringUtils;
25 import org.codelibs.core.collection.LruHashMap;
26 import org.codelibs.core.lang.StringUtil;
27 import org.codelibs.fess.Constants;
28 import org.codelibs.fess.crawler.entity.AccessResult;
29 import org.codelibs.fess.crawler.entity.AccessResultData;
30 import org.codelibs.fess.crawler.entity.UrlQueue;
31 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
32 import org.codelibs.fess.mylasta.direction.FessConfig;
33 import org.codelibs.fess.util.ComponentUtil;
34 import org.codelibs.fess.util.GroovyUtil;
35 import org.slf4j.Logger;
36
37 public interface FessTransformer {
38
39 public static Map<String, String> parentEncodingMap = Collections.synchronizedMap(new LruHashMap<String, String>(1000));
40
41 FessConfig getFessConfig();
42
43 Logger getLogger();
44
45 public default String getHost(final String u) {
46 if (StringUtil.isBlank(u)) {
47 return StringUtil.EMPTY;
48 }
49
50 String url = u;
51 final String originalUrl = url;
52
53 int idx = url.indexOf("://");
54 if (idx >= 0) {
55 url = url.substring(idx + 3);
56 }
57
58 idx = url.indexOf('/');
59 if (idx >= 0) {
60 url = url.substring(0, idx);
61 }
62
63 if (url.equals(originalUrl)) {
64 return getFessConfig().getCrawlerDocumentUnknownHostname();
65 }
66
67 return url;
68 }
69
70 public default String getSite(final String u, final String encoding) {
71 if (StringUtil.isBlank(u)) {
72 return StringUtil.EMPTY;
73 }
74
75 String url = u;
76 int idx = url.indexOf("://");
77 if (idx >= 0) {
78 url = url.substring(idx + 3);
79 }
80
81 idx = url.indexOf('?');
82 if (idx >= 0) {
83 url = url.substring(0, idx);
84 }
85
86 if (encoding != null) {
87 String enc;
88 if (StringUtil.isNotBlank(getFessConfig().getCrawlerDocumentSiteEncoding())) {
89 if (getFessConfig().isCrawlerDocumentUseSiteEncodingOnEnglish()) {
90 if ("ISO-8859-1".equalsIgnoreCase(encoding) || "US-ASCII".equalsIgnoreCase(encoding)) {
91 enc = getFessConfig().getCrawlerDocumentSiteEncoding();
92 } else {
93 enc = encoding;
94 }
95 } else {
96 enc = getFessConfig().getCrawlerDocumentSiteEncoding();
97 }
98 } else {
99 enc = encoding;
100 }
101
102 try {
103 url = URLDecoder.decode(url, enc);
104 } catch (final Exception e) {}
105 }
106
107 return StringUtils.abbreviate(url, getMaxSiteLength());
108 }
109
110 public default void putResultDataBody(final Map<String, Object> dataMap, final String key, final Object value) {
111 final FessConfig fessConfig = ComponentUtil.getFessConfig();
112 if (fessConfig.getIndexFieldUrl().equals(key)) {
113 dataMap.put(key, value);
114 } else if (dataMap.containsKey(key)) {
115 if (getFessConfig().isCrawlerDocumentAppendData()) {
116 final Object oldValue = dataMap.get(key);
117 final Object[] oldValues = (Object[]) oldValue;
118 if (value.getClass().isArray()) {
119 final Object[] newValues = (Object[]) value;
120 final Object[] values = Arrays.copyOf(oldValues, oldValues.length + newValues.length);
121 for (int i = 0; i < newValues.length; i++) {
122 values[values.length - 1 + i] = newValues[i];
123 }
124 dataMap.put(key, values);
125 } else {
126 final Object[] values = Arrays.copyOf(oldValues, oldValues.length + 1);
127 values[values.length - 1] = value;
128 dataMap.put(key, values);
129 }
130 } else {
131 dataMap.put(key, value);
132 }
133 } else {
134 dataMap.put(key, value);
135 }
136 }
137
138 public default void putResultDataWithTemplate(final Map<String, Object> dataMap, final String key, final Object value,
139 final String template) {
140 Object target = value;
141 if (template != null) {
142 final Map<String, Object> contextMap = new HashMap<>();
143 contextMap.put("doc", dataMap);
144 final Map<String, Object> paramMap = new HashMap<>(dataMap.size() + 2);
145 paramMap.putAll(dataMap);
146 paramMap.put("value", target);
147 paramMap.put("context", contextMap);
148 target = evaluateValue(template, paramMap);
149 }
150 if (key != null && target != null) {
151 if (target.getClass().isArray()) {
152 if (key.endsWith("_m")) {
153 putResultDataBody(dataMap, key, target);
154 } else {
155 putResultDataBody(dataMap, key, Arrays.toString((Object[]) target));
156 }
157 } else {
158 if (key.endsWith("_m")) {
159 putResultDataBody(dataMap, key, new Object[] { target });
160 } else {
161 putResultDataBody(dataMap, key, target);
162 }
163 }
164 }
165 }
166
167 public default String evaluateValue(final String template, final Map<String, Object> paramMap) {
168 if (StringUtil.isEmpty(template)) {
169 return StringUtil.EMPTY;
170 }
171
172 final Object value = GroovyUtil.evaluate(template, paramMap);
173 if (value == null) {
174 return null;
175 }
176 return value.toString();
177 }
178
179 public default int getMaxSiteLength() {
180 return getFessConfig().getCrawlerDocumentMaxSiteLengthAsInteger();
181 }
182
183 public default String getFileName(final String url, final String encoding) {
184 if (StringUtil.isBlank(url)) {
185 return StringUtil.EMPTY;
186 }
187
188 String u = url;
189 int idx = u.lastIndexOf('?');
190 if (idx >= 0) {
191 u = u.substring(0, idx);
192 }
193
194 idx = u.lastIndexOf('#');
195 if (idx >= 0) {
196 u = u.substring(0, idx);
197 }
198 u = decodeUrlAsName(u, u.startsWith("file:"));
199 idx = u.lastIndexOf('/');
200 if (idx >= 0) {
201 if (u.length() > idx + 1) {
202 u = u.substring(idx + 1);
203 } else {
204 u = StringUtil.EMPTY;
205 }
206 }
207
208 try {
209 u = URLDecoder.decode(u, encoding);
210 } catch (final Exception e) {
211
212 }
213 return u;
214 }
215
216 public default String decodeUrlAsName(final String url, final boolean escapePlus) {
217 if (url == null) {
218 return null;
219 }
220
221 final FessConfig fessConfig = getFessConfig();
222 String enc = Constants.UTF_8;
223 if (StringUtil.isBlank(fessConfig.getCrawlerDocumentFileNameEncoding())) {
224 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
225 if (urlQueue != null) {
226 final String parentUrl = urlQueue.getParentUrl();
227 if (StringUtil.isNotEmpty(parentUrl)) {
228 final String sessionId = urlQueue.getSessionId();
229 final String pageEnc = getParentEncoding(parentUrl, sessionId);
230 if (pageEnc != null) {
231 enc = pageEnc;
232 } else if (urlQueue.getEncoding() != null) {
233 enc = urlQueue.getEncoding();
234 }
235 }
236 }
237 } else {
238 enc = fessConfig.getCrawlerDocumentFileNameEncoding();
239 }
240
241 final String escapedUrl = escapePlus ? url.replace("+", "%2B") : url;
242 try {
243 return URLDecoder.decode(escapedUrl, enc);
244 } catch (final Exception e) {
245 return url;
246 }
247 }
248
249 public default String getParentEncoding(final String parentUrl, final String sessionId) {
250 final String key = sessionId + ":" + parentUrl;
251 String enc = parentEncodingMap.get(key);
252 if (enc != null) {
253 return enc;
254 }
255
256 final AccessResult<?> accessResult = ComponentUtil.getDataService().getAccessResult(sessionId, parentUrl);
257 if (accessResult != null) {
258 final AccessResultData<?> accessResultData = accessResult.getAccessResultData();
259 if (accessResultData != null && accessResultData.getEncoding() != null) {
260 enc = accessResultData.getEncoding();
261 parentEncodingMap.put(key, enc);
262 return enc;
263 }
264 }
265 return null;
266 }
267 }