1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import java.net.URLDecoder;
19 import java.util.Arrays;
20 import java.util.Collection;
21 import java.util.Collections;
22 import java.util.HashMap;
23 import java.util.Map;
24
25 import org.apache.commons.lang3.StringUtils;
26 import org.apache.logging.log4j.Logger;
27 import org.codelibs.core.collection.LruHashMap;
28 import org.codelibs.core.lang.StringUtil;
29 import org.codelibs.fess.Constants;
30 import org.codelibs.fess.crawler.entity.AccessResult;
31 import org.codelibs.fess.crawler.entity.AccessResultData;
32 import org.codelibs.fess.crawler.entity.UrlQueue;
33 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
34 import org.codelibs.fess.mylasta.direction.FessConfig;
35 import org.codelibs.fess.util.ComponentUtil;
36
37 public interface FessTransformer {
38
39 Map<String, String> parentEncodingMap = Collections.synchronizedMap(new LruHashMap<String, String>(1000));
40
41 FessConfig getFessConfig();
42
43 Logger getLogger();
44
45 default String getHost(final String u) {
46 if (StringUtil.isBlank(u)) {
47 return StringUtil.EMPTY;
48 }
49
50 String url = u;
51 final String originalUrl = url;
52
53 int idx = url.indexOf("://");
54 if (idx >= 0) {
55 url = url.substring(idx + 3);
56 }
57
58 idx = url.indexOf('/');
59 if (idx >= 0) {
60 url = url.substring(0, idx);
61 }
62
63 if (url.equals(originalUrl)) {
64 return getFessConfig().getCrawlerDocumentUnknownHostname();
65 }
66
67 return url;
68 }
69
70 default String getSite(final String u, final String encoding) {
71 if (StringUtil.isBlank(u)) {
72 return StringUtil.EMPTY;
73 }
74
75 String url = u;
76 int idx = url.indexOf("://");
77 if (idx >= 0) {
78 url = url.substring(idx + 3);
79 }
80
81 idx = url.indexOf('?');
82 if (idx >= 0) {
83 url = url.substring(0, idx);
84 }
85
86 if (encoding != null) {
87 String enc;
88 if (StringUtil.isNotBlank(getFessConfig().getCrawlerDocumentSiteEncoding())) {
89 if (getFessConfig().isCrawlerDocumentUseSiteEncodingOnEnglish()) {
90 if ("ISO-8859-1".equalsIgnoreCase(encoding) || "US-ASCII".equalsIgnoreCase(encoding)) {
91 enc = getFessConfig().getCrawlerDocumentSiteEncoding();
92 } else {
93 enc = encoding;
94 }
95 } else {
96 enc = getFessConfig().getCrawlerDocumentSiteEncoding();
97 }
98 } else {
99 enc = encoding;
100 }
101
102 try {
103 url = URLDecoder.decode(url, enc);
104 } catch (final Exception e) {}
105 }
106
107 return abbreviateSite(url);
108 }
109
110 default void putResultDataBody(final Map<String, Object> dataMap, final String key, final Object value) {
111 final FessConfig fessConfig = ComponentUtil.getFessConfig();
112 if (fessConfig.getIndexFieldUrl().equals(key) || !dataMap.containsKey(key) || !getFessConfig().isCrawlerDocumentAppendData()) {
113 dataMap.put(key, value);
114 } else {
115 final Object oldValue = dataMap.get(key);
116 final Object[] oldValues;
117 if (oldValue instanceof Object[]) {
118 oldValues = (Object[]) oldValue;
119 } else if (oldValue instanceof Collection<?>) {
120 oldValues = ((Collection<?>) oldValue).toArray();
121 } else {
122 oldValues = new Object[] { oldValue };
123 }
124 if (value.getClass().isArray()) {
125 final Object[] newValues = (Object[]) value;
126 final Object[] values = Arrays.copyOf(oldValues, oldValues.length + newValues.length);
127 for (int i = 0; i < newValues.length; i++) {
128 values[values.length - 1 + i] = newValues[i];
129 }
130 dataMap.put(key, values);
131 } else {
132 final Object[] values = Arrays.copyOf(oldValues, oldValues.length + 1);
133 values[values.length - 1] = value;
134 dataMap.put(key, values);
135 }
136 }
137 }
138
139 default void putResultDataWithTemplate(final Map<String, Object> dataMap, final String key, final Object value, final String template,
140 final String scriptType) {
141 Object target = value;
142 if (template != null) {
143 final Map<String, Object> contextMap = new HashMap<>();
144 contextMap.put("doc", dataMap);
145 final Map<String, Object> paramMap = new HashMap<>(dataMap.size() + 2);
146 paramMap.putAll(dataMap);
147 paramMap.put("value", target);
148 paramMap.put("context", contextMap);
149 target = evaluateValue(scriptType, template, paramMap);
150 }
151 if (key != null && target != null) {
152 putResultDataBody(dataMap, key, target);
153 }
154 }
155
156 default Object evaluateValue(final String scriptType, String template, final Map<String, Object> paramMap) {
157 if (StringUtil.isEmpty(template)) {
158 return StringUtil.EMPTY;
159 }
160
161 return ComponentUtil.getScriptEngineFactory().getScriptEngine(scriptType).evaluate(template, paramMap);
162 }
163
164 default int getMaxSiteLength() {
165 return getFessConfig().getCrawlerDocumentMaxSiteLengthAsInteger();
166 }
167
168 default String abbreviateSite(final String value) {
169 final int maxSiteLength = getMaxSiteLength();
170 if (maxSiteLength > -1) {
171 return StringUtils.abbreviate(value, maxSiteLength);
172 }
173 return value;
174 }
175
176 default String getFileName(final String url, final String encoding) {
177 if (StringUtil.isBlank(url)) {
178 return StringUtil.EMPTY;
179 }
180
181 String u = url;
182 int idx = u.lastIndexOf('?');
183 if (idx >= 0) {
184 u = u.substring(0, idx);
185 }
186
187 idx = u.lastIndexOf('#');
188 if (idx >= 0) {
189 u = u.substring(0, idx);
190 }
191 u = decodeUrlAsName(u, u.startsWith("file:"));
192 idx = u.lastIndexOf('/');
193 if (idx >= 0) {
194 if (u.length() > idx + 1) {
195 u = u.substring(idx + 1);
196 } else {
197 u = StringUtil.EMPTY;
198 }
199 }
200 return u;
201 }
202
203 default String decodeUrlAsName(final String url, final boolean escapePlus) {
204 if (url == null) {
205 return null;
206 }
207
208 final FessConfig fessConfig = getFessConfig();
209 String enc = Constants.UTF_8;
210 if (StringUtil.isBlank(fessConfig.getCrawlerDocumentFileNameEncoding())) {
211 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
212 if (urlQueue != null) {
213 final String parentUrl = urlQueue.getParentUrl();
214 if (StringUtil.isNotEmpty(parentUrl)) {
215 final String sessionId = urlQueue.getSessionId();
216 final String pageEnc = getParentEncoding(parentUrl, sessionId);
217 if (pageEnc != null) {
218 enc = pageEnc;
219 } else if (urlQueue.getEncoding() != null) {
220 enc = urlQueue.getEncoding();
221 }
222 }
223 }
224 } else {
225 enc = fessConfig.getCrawlerDocumentFileNameEncoding();
226 }
227
228 final String escapedUrl = escapePlus ? url.replace("+", "%2B") : url;
229 try {
230 return URLDecoder.decode(escapedUrl, enc);
231 } catch (final Exception e) {
232 return url;
233 }
234 }
235
236 default String getParentEncoding(final String parentUrl, final String sessionId) {
237 final String key = sessionId + ":" + parentUrl;
238 String enc = parentEncodingMap.get(key);
239 if (enc != null) {
240 return enc;
241 }
242
243 final AccessResult<?> accessResult = ComponentUtil.getDataService().getAccessResult(sessionId, parentUrl);
244 if (accessResult != null) {
245 final AccessResultData<?> accessResultData = accessResult.getAccessResultData();
246 if (accessResultData != null && accessResultData.getEncoding() != null) {
247 enc = accessResultData.getEncoding();
248 parentEncodingMap.put(key, enc);
249 return enc;
250 }
251 }
252 return null;
253 }
254 }