1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.InputStream;
21 import java.net.URLDecoder;
22 import java.util.ArrayList;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.Set;
29
30 import org.apache.commons.lang3.StringUtils;
31 import org.apache.tika.metadata.HttpHeaders;
32 import org.apache.tika.metadata.TikaMetadataKeys;
33 import org.codelibs.core.io.SerializeUtil;
34 import org.codelibs.core.lang.StringUtil;
35 import org.codelibs.core.misc.Pair;
36 import org.codelibs.fess.Constants;
37 import org.codelibs.fess.crawler.client.smb.SmbClient;
38 import org.codelibs.fess.crawler.entity.AccessResultData;
39 import org.codelibs.fess.crawler.entity.ExtractData;
40 import org.codelibs.fess.crawler.entity.ResponseData;
41 import org.codelibs.fess.crawler.entity.ResultData;
42 import org.codelibs.fess.crawler.entity.UrlQueue;
43 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
44 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
45 import org.codelibs.fess.crawler.extractor.Extractor;
46 import org.codelibs.fess.crawler.transformer.impl.AbstractTransformer;
47 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
48 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
49 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
50 import org.codelibs.fess.helper.CrawlingConfigHelper;
51 import org.codelibs.fess.helper.CrawlingInfoHelper;
52 import org.codelibs.fess.helper.DocumentHelper;
53 import org.codelibs.fess.helper.FileTypeHelper;
54 import org.codelibs.fess.helper.LabelTypeHelper;
55 import org.codelibs.fess.helper.PathMappingHelper;
56 import org.codelibs.fess.helper.SambaHelper;
57 import org.codelibs.fess.helper.SystemHelper;
58 import org.codelibs.fess.mylasta.direction.FessConfig;
59 import org.codelibs.fess.util.ComponentUtil;
60 import org.slf4j.Logger;
61 import org.slf4j.LoggerFactory;
62
63 import jcifs.smb.ACE;
64 import jcifs.smb.SID;
65
66 public abstract class AbstractFessFileTransformer extends AbstractTransformer implements FessTransformer {
67
68 private static final Logger logger = LoggerFactory.getLogger(AbstractFessFileTransformer.class);
69
70 protected Map<String, String> metaContentMapping;
71
72 protected FessConfig fessConfig;
73
74 protected abstract Extractor getExtractor(ResponseData responseData);
75
76 @Override
77 public ResultData transform(final ResponseData responseData) {
78 if (responseData == null || !responseData.hasResponseBody()) {
79 throw new CrawlingAccessException("No response body.");
80 }
81
82 final ResultData resultData = new ResultData();
83 resultData.setTransformerName(getName());
84 try {
85 resultData.setData(SerializeUtil.fromObjectToBinary(generateData(responseData)));
86 } catch (final Exception e) {
87 throw new CrawlingAccessException("Could not serialize object", e);
88 }
89 resultData.setEncoding(fessConfig.getCrawlerCrawlingDataEncoding());
90
91 return resultData;
92 }
93
94 protected Map<String, Object> generateData(final ResponseData responseData) {
95 final Extractor extractor = getExtractor(responseData);
96 final Map<String, String> params = new HashMap<>();
97 params.put(TikaMetadataKeys.RESOURCE_NAME_KEY, getResourceName(responseData));
98 final String mimeType = responseData.getMimeType();
99 params.put(HttpHeaders.CONTENT_TYPE, mimeType);
100 params.put(HttpHeaders.CONTENT_ENCODING, responseData.getCharSet());
101 params.put(ExtractData.URL, responseData.getUrl());
102 final StringBuilder contentMetaBuf = new StringBuilder(1000);
103 final Map<String, Object> dataMap = new HashMap<>();
104 final Map<String, Object> metaDataMap = new HashMap<>();
105 String content;
106 try (final InputStream in = responseData.getResponseBody()) {
107 final ExtractData extractData = getExtractData(extractor, in, params);
108 content = extractData.getContent();
109 if (fessConfig.isCrawlerDocumentFileIgnoreEmptyContent() && StringUtil.isBlank(content)) {
110 return null;
111 }
112 if (getLogger().isDebugEnabled()) {
113 getLogger().debug("ExtractData: " + extractData);
114 }
115
116 extractData.getKeySet().stream()
117 .filter(k -> extractData.getValues(k) != null)
118 .forEach(key -> {
119 final String[] values = extractData.getValues(key);
120 metaDataMap.put(key, values);
121
122
123 if (fessConfig.isCrawlerMetadataContentIncluded(key)) {
124 final String joinedValue = StringUtils.join(values, ' ');
125 if (StringUtil.isNotBlank(joinedValue)) {
126 if (contentMetaBuf.length() > 0) {
127 contentMetaBuf.append(' ');
128 }
129 contentMetaBuf.append(joinedValue.trim());
130 }
131 }
132
133 final Pair<String, String> mapping = fessConfig.getCrawlerMetadataNameMapping(key);
134 if (mapping != null) {
135 if (Constants.MAPPING_TYPE_ARRAY.equalsIgnoreCase(mapping.getSecond())) {
136 dataMap.put(mapping.getFirst(), values);
137 } else if (Constants.MAPPING_TYPE_STRING.equalsIgnoreCase(mapping.getSecond())) {
138 final String joinedValue = StringUtils.join(values, ' ');
139 dataMap.put(mapping.getFirst(), joinedValue.trim());
140 } else if (values.length == 1) {
141 try {
142 if (Constants.MAPPING_TYPE_LONG.equalsIgnoreCase(mapping.getSecond())) {
143 dataMap.put(mapping.getFirst(), Long.parseLong(values[0]));
144 } else if (Constants.MAPPING_TYPE_DOUBLE.equalsIgnoreCase(mapping.getSecond())) {
145 dataMap.put(mapping.getFirst(), Double.parseDouble(values[0]));
146 } else {
147 logger.warn("Unknown mapping type: {}={}", key, mapping);
148 }
149 } catch (final NumberFormatException e) {
150 logger.warn("Failed to parse " + values[0], e);
151 }
152 }
153 }
154
155 });
156 } catch (final Exception e) {
157 final CrawlingAccessException rcae = new CrawlingAccessException("Could not get a text from " + responseData.getUrl(), e);
158 rcae.setLogLevel(CrawlingAccessException.WARN);
159 throw rcae;
160 }
161 if (content == null) {
162 content = StringUtil.EMPTY;
163 }
164 final String contentMeta = contentMetaBuf.toString().trim();
165
166 final FessConfig fessConfig = ComponentUtil.getFessConfig();
167 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
168 final String sessionId = crawlingInfoHelper.getCanonicalSessionId(responseData.getSessionId());
169 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
170 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
171 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
172 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
173 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
174 final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
175 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
176 String url = responseData.getUrl();
177 final String indexingTarget = crawlingConfig.getIndexingTarget(url);
178 url = pathMappingHelper.replaceUrl(sessionId, url);
179
180 final Map<String, String> fieldConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.FIELD);
181
182 String urlEncoding;
183 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
184 if (urlQueue != null && urlQueue.getEncoding() != null) {
185 urlEncoding = urlQueue.getEncoding();
186 } else {
187 urlEncoding = responseData.getCharSet();
188 }
189
190
191 final String configId = crawlingConfig.getConfigId();
192 if (configId != null) {
193 putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
194 }
195
196 if (documentExpires != null) {
197 putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), documentExpires);
198 }
199
200 putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
201
202 final StringBuilder buf = new StringBuilder(content.length() + 1000);
203 if (fessConfig.isCrawlerDocumentFileAppendBodyContent()) {
204 buf.append(content);
205 }
206 if (fessConfig.isCrawlerDocumentFileAppendMetaContent()) {
207 if (buf.length() > 0) {
208 buf.append(' ');
209 }
210 buf.append(contentMeta);
211 }
212 final String bodyBase = buf.toString().trim();
213 final String body = documentHelper.getContent(responseData, bodyBase, dataMap);
214 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);
215 if ((Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache())) || fessConfig
216 .isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
217 if (responseData.getContentLength() > 0
218 && responseData.getContentLength() <= fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger().longValue()) {
219
220 final String cache = content.trim().replaceAll("[ \\t\\x0B\\f]+", " ");
221
222 putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), cache);
223 putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
224 }
225 }
226
227 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
228 documentHelper.getDigest(responseData, bodyBase, dataMap, fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));
229
230 final String fileName = getFileName(url, urlEncoding);
231 if (!dataMap.containsKey(fessConfig.getIndexFieldTitle())) {
232 if (url.endsWith("/")) {
233 if (StringUtil.isNotBlank(content)) {
234 putResultDataBody(
235 dataMap,
236 fessConfig.getIndexFieldTitle(),
237 documentHelper.getDigest(responseData, body, dataMap,
238 fessConfig.getCrawlerDocumentFileMaxTitleLengthAsInteger()));
239 } else {
240 putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), fessConfig.getCrawlerDocumentFileNoTitleLabel());
241 }
242 } else {
243 if (StringUtil.isBlank(fileName)) {
244 putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), decodeUrlAsName(url, url.startsWith("file:")));
245 } else {
246 putResultDataBody(dataMap, fessConfig.getIndexFieldTitle(), fileName);
247 }
248 }
249 }
250
251 putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHostOnFile(url));
252
253 putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSiteOnFile(url, urlEncoding));
254
255 if (StringUtil.isNotBlank(fileName)) {
256 putResultDataBody(dataMap, fessConfig.getIndexFieldFilename(), fileName);
257 }
258
259 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
260
261 final Date now = systemHelper.getCurrentTime();
262 putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), now);
263
264 putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), StringUtil.EMPTY);
265
266 putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
267 if (fileTypeHelper != null) {
268
269 putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
270 }
271
272 putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
273
274 final Date lastModified = responseData.getLastModified();
275 if (lastModified != null) {
276 putResultDataBody(dataMap, fessConfig.getIndexFieldLastModified(), lastModified);
277
278 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), lastModified);
279 } else {
280
281 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), now);
282 }
283
284 putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
285
286 putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
287
288 final Set<String> labelTypeSet = new HashSet<>();
289 for (final String labelType : crawlingConfig.getLabelTypeValues()) {
290 labelTypeSet.add(labelType);
291 }
292 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
293 labelTypeSet.addAll(labelTypeHelper.getMatchedLabelValueSet(url));
294 putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeSet);
295
296 final List<String> roleTypeList = getRoleTypes(responseData);
297 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
298 putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
299
300 putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
301 stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n])));
302
303
304 if (StringUtil.isNotBlank(fessConfig.getCrawlerDocumentFileDefaultLang())) {
305 putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), fessConfig.getCrawlerDocumentFileDefaultLang());
306 }
307
308 putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
309
310 String parentUrl = responseData.getParentUrl();
311 if (StringUtil.isNotBlank(parentUrl)) {
312 parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
313 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
314 putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
315 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
316 }
317
318 putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), responseData.getUrl());
319
320
321 final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);
322 final Map<String, String> metaConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.META);
323 for (final Map.Entry<String, String> entry : metaConfigMap.entrySet()) {
324 final String key = entry.getKey();
325 final String[] values = entry.getValue().split(",");
326 for (final String value : values) {
327 putResultDataWithTemplate(dataMap, key, metaDataMap.get(value), scriptConfigMap.get(key));
328 }
329 }
330 final Map<String, String> valueConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.VALUE);
331 for (final Map.Entry<String, String> entry : valueConfigMap.entrySet()) {
332 final String key = entry.getKey();
333 putResultDataWithTemplate(dataMap, key, entry.getValue(), scriptConfigMap.get(key));
334 }
335
336 return dataMap;
337 }
338
339 private ExtractData getExtractData(final Extractor extractor, final InputStream in, final Map<String, String> params) {
340 try {
341 return extractor.getText(in, params);
342 } catch (final RuntimeException e) {
343 if (!fessConfig.isCrawlerIgnoreContentException()) {
344 throw e;
345 }
346 if (logger.isDebugEnabled()) {
347 logger.debug("Could not get a text.", e);
348 }
349 }
350 return new ExtractData();
351 }
352
353 private String getResourceName(final ResponseData responseData) {
354 String name = responseData.getUrl();
355 final String enc = responseData.getCharSet();
356
357 if (name == null || enc == null) {
358 return null;
359 }
360
361 name = name.replaceAll("/+$", StringUtil.EMPTY);
362 final int idx = name.lastIndexOf('/');
363 if (idx >= 0) {
364 name = name.substring(idx + 1);
365 }
366 try {
367 return URLDecoder.decode(name, enc);
368 } catch (final Exception e) {
369 return name;
370 }
371 }
372
373 protected String getHostOnFile(final String url) {
374 if (StringUtil.isBlank(url)) {
375 return StringUtil.EMPTY;
376 }
377
378 if (url.startsWith("file:////")) {
379 final String value = decodeUrlAsName(url.substring(9), true);
380 final int pos = value.indexOf('/');
381 if (pos > 0) {
382 return value.substring(0, pos);
383 } else if (pos == -1) {
384 return value;
385 } else {
386 return "localhost";
387 }
388 } else if (url.startsWith("file:")) {
389 return "localhost";
390 }
391
392 return getHost(url);
393 }
394
395 protected List<String> getRoleTypes(final ResponseData responseData) {
396 final List<String> roleTypeList = new ArrayList<>();
397
398 if (fessConfig.isSmbRoleFromFile() && responseData.getUrl().startsWith("smb://")) {
399 final SambaHelper sambaHelper = ComponentUtil.getSambaHelper();
400 final ACE[] aces = (ACE[]) responseData.getMetaDataMap().get(SmbClient.SMB_ACCESS_CONTROL_ENTRIES);
401 if (aces != null) {
402 for (final ACE item : aces) {
403 final SID sid = item.getSID();
404 final String accountId = sambaHelper.getAccountId(sid);
405 if (accountId != null) {
406 roleTypeList.add(accountId);
407 }
408 }
409 if (getLogger().isDebugEnabled()) {
410 getLogger().debug("smbUrl:" + responseData.getUrl() + " roleType:" + roleTypeList.toString());
411 }
412 }
413 }
414
415 return roleTypeList;
416 }
417
418 protected String getSiteOnFile(final String url, final String encoding) {
419 if (StringUtil.isBlank(url)) {
420 return StringUtil.EMPTY;
421 }
422
423 if (url.startsWith("file:////")) {
424 final String value = decodeUrlAsName(url.substring(9), true);
425 return StringUtils.abbreviate("\\\\" + value.replace('/', '\\'), getMaxSiteLength());
426 } else if (url.startsWith("file:")) {
427 final String value = decodeUrlAsName(url.substring(5), true);
428 if (value.length() > 2 && value.charAt(2) == ':') {
429
430 return StringUtils.abbreviate(value.substring(1).replace('/', '\\'), getMaxSiteLength());
431 } else {
432
433 return StringUtils.abbreviate(value, getMaxSiteLength());
434 }
435 }
436
437 return getSite(url, encoding);
438 }
439
440 @Override
441 public Object getData(final AccessResultData<?> accessResultData) {
442 final byte[] data = accessResultData.getData();
443 if (data != null) {
444 try {
445 return SerializeUtil.fromBinaryToObject(data);
446 } catch (final Exception e) {
447 throw new CrawlerSystemException("Could not create an instanced from bytes.", e);
448 }
449 }
450 return new HashMap<String, Object>();
451 }
452
453 public void addMetaContentMapping(final String metaname, final String dynamicField) {
454 if (metaContentMapping == null) {
455 metaContentMapping = new HashMap<>();
456 }
457 metaContentMapping.put(metaname, dynamicField);
458 }
459
460 }