1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.InputStream;
21 import java.net.URLDecoder;
22 import java.util.ArrayList;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.stream.Collectors;
28
29 import org.apache.commons.lang3.StringUtils;
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.apache.tika.metadata.HttpHeaders;
33 import org.apache.tika.metadata.TikaMetadataKeys;
34 import org.codelibs.core.io.SerializeUtil;
35 import org.codelibs.core.lang.StringUtil;
36 import org.codelibs.core.misc.Tuple3;
37 import org.codelibs.fess.Constants;
38 import org.codelibs.fess.crawler.entity.AccessResultData;
39 import org.codelibs.fess.crawler.entity.ExtractData;
40 import org.codelibs.fess.crawler.entity.ResponseData;
41 import org.codelibs.fess.crawler.entity.ResultData;
42 import org.codelibs.fess.crawler.entity.UrlQueue;
43 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
44 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
45 import org.codelibs.fess.crawler.extractor.Extractor;
46 import org.codelibs.fess.crawler.extractor.impl.TikaExtractor;
47 import org.codelibs.fess.crawler.transformer.impl.AbstractTransformer;
48 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
49 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
50 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
51 import org.codelibs.fess.es.config.exentity.CrawlingConfig.Param.Config;
52 import org.codelibs.fess.helper.CrawlingConfigHelper;
53 import org.codelibs.fess.helper.CrawlingInfoHelper;
54 import org.codelibs.fess.helper.DocumentHelper;
55 import org.codelibs.fess.helper.FileTypeHelper;
56 import org.codelibs.fess.helper.LabelTypeHelper;
57 import org.codelibs.fess.helper.PathMappingHelper;
58 import org.codelibs.fess.helper.PermissionHelper;
59 import org.codelibs.fess.helper.SystemHelper;
60 import org.codelibs.fess.mylasta.direction.FessConfig;
61 import org.codelibs.fess.taglib.FessFunctions;
62 import org.codelibs.fess.util.ComponentUtil;
63
64 public abstract class AbstractFessFileTransformer extends AbstractTransformer implements FessTransformer {
65
66 private static final Logger logger = LogManager.getLogger(AbstractFessFileTransformer.class);
67
68 protected Map<String, String> metaContentMapping;
69
70 protected FessConfig fessConfig;
71
72 protected abstract Extractor getExtractor(ResponseData responseData);
73
74 @Override
75 public ResultData transform(final ResponseData responseData) {
76 if (responseData == null || !responseData.hasResponseBody()) {
77 throw new CrawlingAccessException("No response body.");
78 }
79
80 final ResultData resultData = new ResultData();
81 resultData.setTransformerName(getName());
82 try {
83 resultData.setData(SerializeUtil.fromObjectToBinary(generateData(responseData)));
84 } catch (final Exception e) {
85 throw new CrawlingAccessException("Could not serialize object", e);
86 }
87 resultData.setEncoding(fessConfig.getCrawlerCrawlingDataEncoding());
88
89 return resultData;
90 }
91
92 protected Map<String, Object> generateData(final ResponseData responseData) {
93 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
94 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
95 final Extractor extractor = getExtractor(responseData);
96 final String mimeType = responseData.getMimeType();
97 final StringBuilder contentMetaBuf = new StringBuilder(1000);
98 final Map<String, Object> dataMap = new HashMap<>();
99 final Map<String, Object> metaDataMap = new HashMap<>();
100 String content;
101 try (final InputStream in = responseData.getResponseBody()) {
102 final ExtractData extractData = getExtractData(extractor, in, createExtractParams(responseData, crawlingConfig));
103 content = extractData.getContent();
104 if (fessConfig.isCrawlerDocumentFileIgnoreEmptyContent() && StringUtil.isBlank(content)) {
105 return null;
106 }
107 if (getLogger().isDebugEnabled()) {
108 getLogger().debug("ExtractData: {}", extractData);
109 }
110
111 extractData.getKeySet().stream().filter(k -> extractData.getValues(k) != null).forEach(key -> {
112 final String[] values = extractData.getValues(key);
113 metaDataMap.put(key, values);
114
115
116 if (fessConfig.isCrawlerMetadataContentIncluded(key)) {
117 final String joinedValue = StringUtils.join(values, ' ');
118 if (StringUtil.isNotBlank(joinedValue)) {
119 if (contentMetaBuf.length() > 0) {
120 contentMetaBuf.append(' ');
121 }
122 contentMetaBuf.append(joinedValue.trim());
123 }
124 }
125
126 final Tuple3<String, String, String> mapping = fessConfig.getCrawlerMetadataNameMapping(key);
127 if (mapping != null) {
128 if (Constants.MAPPING_TYPE_ARRAY.equalsIgnoreCase(mapping.getValue2())) {
129 dataMap.put(mapping.getValue1(), values);
130 } else if (Constants.MAPPING_TYPE_STRING.equalsIgnoreCase(mapping.getValue2())) {
131 final String joinedValue = StringUtils.join(values, ' ');
132 dataMap.put(mapping.getValue1(), joinedValue.trim());
133 } else if (values.length == 1) {
134 try {
135 if (Constants.MAPPING_TYPE_LONG.equalsIgnoreCase(mapping.getValue2())) {
136 dataMap.put(mapping.getValue1(), Long.parseLong(values[0]));
137 } else if (Constants.MAPPING_TYPE_DOUBLE.equalsIgnoreCase(mapping.getValue2())) {
138 dataMap.put(mapping.getValue1(), Double.parseDouble(values[0]));
139 } else if (Constants.MAPPING_TYPE_DATE.equalsIgnoreCase(mapping.getValue2())
140 || Constants.MAPPING_TYPE_PDF_DATE.equalsIgnoreCase(mapping.getValue2())) {
141 final String dateFormate;
142 if (StringUtil.isNotBlank(mapping.getValue3())) {
143 dateFormate = mapping.getValue3();
144 } else if (Constants.MAPPING_TYPE_PDF_DATE.equalsIgnoreCase(mapping.getValue2())) {
145 dateFormate = mapping.getValue2();
146 } else {
147 dateFormate = Constants.DATE_OPTIONAL_TIME;
148 }
149 final Date dt = FessFunctions.parseDate(values[0], dateFormate);
150 if (dt != null) {
151 dataMap.put(mapping.getValue1(), FessFunctions.formatDate(dt));
152 } else {
153 logger.warn("Failed to parse {}", mapping.toString());
154 }
155 } else {
156 logger.warn("Unknown mapping type: {}={}", key, mapping);
157 }
158 } catch (final Exception e) {
159 logger.warn("Failed to parse {}", values[0], e);
160 }
161 }
162 }
163
164 });
165 } catch (final Exception e) {
166 final CrawlingAccessException rcae = new CrawlingAccessException("Could not get a text from " + responseData.getUrl(), e);
167 rcae.setLogLevel(CrawlingAccessException.WARN);
168 throw rcae;
169 }
170 if (content == null) {
171 content = StringUtil.EMPTY;
172 }
173 final String contentMeta = contentMetaBuf.toString().trim();
174
175 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
176 final String sessionId = crawlingInfoHelper.getCanonicalSessionId(responseData.getSessionId());
177 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
178 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
179 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
180 final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
181 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
182 String url = responseData.getUrl();
183 final String indexingTarget = crawlingConfig.getIndexingTarget(url);
184 url = pathMappingHelper.replaceUrl(sessionId, url);
185
186 final Map<String, String> fieldConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.FIELD);
187
188 String urlEncoding;
189 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
190 if (urlQueue != null && urlQueue.getEncoding() != null) {
191 urlEncoding = urlQueue.getEncoding();
192 } else {
193 urlEncoding = responseData.getCharSet();
194 }
195
196
197 final String configId = crawlingConfig.getConfigId();
198 if (configId != null) {
199 putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
200 }
201
202 if (documentExpires != null) {
203 putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), documentExpires);
204 }
205
206 putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
207
208 final StringBuilder buf = new StringBuilder(content.length() + 1000);
209 if (fessConfig.isCrawlerDocumentFileAppendBodyContent()) {
210 buf.append(content);
211 }
212 if (fessConfig.isCrawlerDocumentFileAppendMetaContent()) {
213 if (buf.length() > 0) {
214 buf.append(' ');
215 }
216 buf.append(contentMeta);
217 }
218 final String bodyBase = buf.toString().trim();
219 responseData.addMetaData(Extractor.class.getSimpleName(), extractor);
220 final String body = documentHelper.getContent(crawlingConfig, responseData, bodyBase, dataMap);
221 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);
222 if ((Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache()))
223 || fessConfig.isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
224 if (responseData.getContentLength() > 0
225 && responseData.getContentLength() <= fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger().longValue()) {
226
227 final String cache = content.trim().replaceAll("[ \\t\\x0B\\f]+", " ");
228
229 putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), cache);
230 putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
231 }
232 }
233
234 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
235 documentHelper.getDigest(responseData, bodyBase, dataMap, fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));
236
237 final String fileName = getFileName(url, urlEncoding);
238 if (!hasTitle(dataMap)) {
239 final String titleField = fessConfig.getIndexFieldTitle();
240 dataMap.remove(titleField);
241 if (url.endsWith("/")) {
242 if (StringUtil.isNotBlank(content)) {
243 putResultDataBody(dataMap, titleField, documentHelper.getDigest(responseData, body, dataMap,
244 fessConfig.getCrawlerDocumentFileMaxTitleLengthAsInteger()));
245 } else {
246 putResultDataBody(dataMap, titleField, fessConfig.getCrawlerDocumentFileNoTitleLabel());
247 }
248 } else if (StringUtil.isBlank(fileName)) {
249 putResultDataBody(dataMap, titleField, decodeUrlAsName(url, url.startsWith("file:")));
250 } else {
251 putResultDataBody(dataMap, titleField, fileName);
252 }
253 }
254
255 putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHostOnFile(url));
256
257 putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSiteOnFile(url, urlEncoding));
258
259 if (StringUtil.isNotBlank(fileName)) {
260 putResultDataBody(dataMap, fessConfig.getIndexFieldFilename(), fileName);
261 }
262
263 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
264
265 final Date now = systemHelper.getCurrentTime();
266 putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), now);
267
268 putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), StringUtil.EMPTY);
269
270 putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
271 if (fileTypeHelper != null) {
272
273 putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
274 }
275
276 putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
277
278 final Date lastModified = getLastModified(dataMap, responseData);
279 if (lastModified != null) {
280 dataMap.put(fessConfig.getIndexFieldLastModified(), lastModified);
281
282 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), lastModified);
283 } else {
284
285 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), now);
286 }
287
288 putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
289
290 putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
291
292 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
293 putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeHelper.getMatchedLabelValueSet(url));
294
295 final List<String> roleTypeList = getRoleTypes(responseData);
296 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
297 putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
298
299 putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
300 stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
301
302
303 if (StringUtil.isNotBlank(fessConfig.getCrawlerDocumentFileDefaultLang())) {
304 putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), fessConfig.getCrawlerDocumentFileDefaultLang());
305 }
306
307 putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
308
309 String parentUrl = responseData.getParentUrl();
310 if (StringUtil.isNotBlank(parentUrl)) {
311 parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
312 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
313 putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
314 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
315 }
316
317 putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), responseData.getUrl());
318
319
320 final String scriptType = crawlingConfig.getScriptType();
321 final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);
322 final Map<String, String> metaConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.META);
323 for (final Map.Entry<String, String> entry : metaConfigMap.entrySet()) {
324 final String key = entry.getKey();
325 final String[] values = entry.getValue().split(",");
326 for (final String value : values) {
327 putResultDataWithTemplate(dataMap, key, metaDataMap.get(value), scriptConfigMap.get(key), scriptType);
328 }
329 }
330 final Map<String, String> valueConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.VALUE);
331 for (final Map.Entry<String, String> entry : valueConfigMap.entrySet()) {
332 final String key = entry.getKey();
333 putResultDataWithTemplate(dataMap, key, entry.getValue(), scriptConfigMap.get(key), scriptType);
334 }
335
336 return dataMap;
337 }
338
339 protected Date getLastModified(final Map<String, Object> dataMap, final ResponseData responseData) {
340 final Object lastModifiedObj = dataMap.get(fessConfig.getIndexFieldLastModified());
341 if (lastModifiedObj instanceof Date) {
342 return (Date) lastModifiedObj;
343 }
344 if (lastModifiedObj instanceof String) {
345 final Date lastModified = FessFunctions.parseDate(lastModifiedObj.toString());
346 if (lastModified != null) {
347 return lastModified;
348 }
349 } else if (lastModifiedObj instanceof String[]) {
350 final String[] lastModifieds = (String[]) lastModifiedObj;
351 if (lastModifieds.length > 0) {
352 final Date lastModified = FessFunctions.parseDate(lastModifieds[0]);
353 if (lastModified != null) {
354 return lastModified;
355 }
356 }
357 }
358
359 return responseData.getLastModified();
360 }
361
362 protected boolean hasTitle(final Map<String, Object> dataMap) {
363 final Object titleObj = dataMap.get(fessConfig.getIndexFieldTitle());
364 if (titleObj != null) {
365 if (titleObj instanceof String[]) {
366 return stream((String[]) titleObj).get(stream -> stream.anyMatch(StringUtil::isNotBlank));
367 }
368 return StringUtil.isNotBlank(titleObj.toString());
369 }
370 return false;
371 }
372
373 protected Map<String, String> createExtractParams(final ResponseData responseData, final CrawlingConfig crawlingConfig) {
374 final Map<String, String> params = new HashMap<>(crawlingConfig.getConfigParameterMap(ConfigName.CONFIG));
375 params.put(TikaMetadataKeys.RESOURCE_NAME_KEY, getResourceName(responseData));
376 params.put(HttpHeaders.CONTENT_TYPE, responseData.getMimeType());
377 params.put(HttpHeaders.CONTENT_ENCODING, responseData.getCharSet());
378 params.put(ExtractData.URL, responseData.getUrl());
379 final Map<String, String> configParam = crawlingConfig.getConfigParameterMap(ConfigName.CONFIG);
380 if (configParam != null) {
381 final String keepOriginalBody = configParam.get(Config.KEEP_ORIGINAL_BODY);
382 if (StringUtil.isNotBlank(keepOriginalBody)) {
383 params.put(TikaExtractor.NORMALIZE_TEXT,
384 Constants.TRUE.equalsIgnoreCase(keepOriginalBody) ? Constants.FALSE : Constants.TRUE);
385 }
386 }
387 return params;
388 }
389
390 protected ExtractData getExtractData(final Extractor extractor, final InputStream in, final Map<String, String> params) {
391 try {
392 return extractor.getText(in, params);
393 } catch (final RuntimeException e) {
394 if (!fessConfig.isCrawlerIgnoreContentException()) {
395 throw e;
396 }
397 if (logger.isDebugEnabled()) {
398 logger.debug("Could not get a text.", e);
399 }
400 }
401 return new ExtractData();
402 }
403
404 protected String getResourceName(final ResponseData responseData) {
405 String name = responseData.getUrl();
406 final String enc = responseData.getCharSet();
407
408 if (name == null || enc == null) {
409 return null;
410 }
411
412 name = name.replaceAll("/+$", StringUtil.EMPTY);
413 final int idx = name.lastIndexOf('/');
414 if (idx >= 0) {
415 name = name.substring(idx + 1);
416 }
417 try {
418 return URLDecoder.decode(name, enc);
419 } catch (final Exception e) {
420 return name;
421 }
422 }
423
424 protected String getHostOnFile(final String url) {
425 if (StringUtil.isBlank(url)) {
426 return StringUtil.EMPTY;
427 }
428
429 if (url.startsWith("file:////")) {
430 final String value = decodeUrlAsName(url.substring(9), true);
431 final int pos = value.indexOf('/');
432 if (pos > 0) {
433 return value.substring(0, pos);
434 }
435 if (pos == -1) {
436 return value;
437 } else {
438 return "localhost";
439 }
440 }
441 if (url.startsWith("file:")) {
442 return "localhost";
443 }
444
445 return getHost(url);
446 }
447
448 protected List<String> getRoleTypes(final ResponseData responseData) {
449 final List<String> roleTypeList = new ArrayList<>();
450 final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
451
452 roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
453 roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
454 roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
455
456 return roleTypeList;
457 }
458
459 protected String getSiteOnFile(final String url, final String encoding) {
460 if (StringUtil.isBlank(url)) {
461 return StringUtil.EMPTY;
462 }
463
464 if (url.startsWith("file:////")) {
465 final String value = decodeUrlAsName(url.substring(9), true);
466 return abbreviateSite("\\\\" + value.replace('/', '\\'));
467 }
468 if (url.startsWith("file:")) {
469 final String value = decodeUrlAsName(url.substring(5), true);
470 if (value.length() > 2 && value.charAt(2) == ':') {
471
472 return abbreviateSite(value.substring(1).replace('/', '\\'));
473 } else {
474
475 return abbreviateSite(value);
476 }
477 }
478 if (url.startsWith("smb:") || url.startsWith("smb1:")) {
479 final String value = url.replaceFirst("^smb.?:/+", StringUtil.EMPTY);
480 return abbreviateSite("\\\\" + value.replace('/', '\\'));
481 }
482
483 return getSite(url, encoding);
484 }
485
486 @Override
487 public Object getData(final AccessResultData<?> accessResultData) {
488 final byte[] data = accessResultData.getData();
489 if (data != null) {
490 try {
491 return SerializeUtil.fromBinaryToObject(data);
492 } catch (final Exception e) {
493 throw new CrawlerSystemException("Could not create an instanced from bytes.", e);
494 }
495 }
496 return new HashMap<String, Object>();
497 }
498
499 public void addMetaContentMapping(final String metaname, final String dynamicField) {
500 if (metaContentMapping == null) {
501 metaContentMapping = new HashMap<>();
502 }
503 metaContentMapping.put(metaname, dynamicField);
504 }
505
506 }