1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.InputStream;
21 import java.net.URLDecoder;
22 import java.util.ArrayList;
23 import java.util.Date;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Map;
27 import java.util.stream.Collectors;
28
29 import org.apache.commons.lang3.StringUtils;
30 import org.apache.logging.log4j.LogManager;
31 import org.apache.logging.log4j.Logger;
32 import org.codelibs.core.lang.StringUtil;
33 import org.codelibs.core.misc.Tuple3;
34 import org.codelibs.fess.Constants;
35 import org.codelibs.fess.crawler.entity.AccessResultData;
36 import org.codelibs.fess.crawler.entity.ExtractData;
37 import org.codelibs.fess.crawler.entity.ResponseData;
38 import org.codelibs.fess.crawler.entity.ResultData;
39 import org.codelibs.fess.crawler.entity.UrlQueue;
40 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
41 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
42 import org.codelibs.fess.crawler.extractor.Extractor;
43 import org.codelibs.fess.crawler.extractor.impl.TikaExtractor;
44 import org.codelibs.fess.crawler.serializer.DataSerializer;
45 import org.codelibs.fess.crawler.transformer.impl.AbstractTransformer;
46 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
47 import org.codelibs.fess.crawler.util.FieldConfigs;
48 import org.codelibs.fess.helper.CrawlingConfigHelper;
49 import org.codelibs.fess.helper.CrawlingInfoHelper;
50 import org.codelibs.fess.helper.DocumentHelper;
51 import org.codelibs.fess.helper.FileTypeHelper;
52 import org.codelibs.fess.helper.LabelTypeHelper;
53 import org.codelibs.fess.helper.PathMappingHelper;
54 import org.codelibs.fess.helper.PermissionHelper;
55 import org.codelibs.fess.helper.SystemHelper;
56 import org.codelibs.fess.mylasta.direction.FessConfig;
57 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig;
58 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.ConfigName;
59 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.Param.Config;
60 import org.codelibs.fess.taglib.FessFunctions;
61 import org.codelibs.fess.util.ComponentUtil;
62
63
64
65
66 public abstract class AbstractFessFileTransformer extends AbstractTransformer implements FessTransformer {
67
68
69
70
71 public AbstractFessFileTransformer() {
72 super();
73 }
74
75 private static final Logger logger = LogManager.getLogger(AbstractFessFileTransformer.class);
76
77
78
79
80 protected Map<String, String> metaContentMapping;
81
82
83
84
85 protected FessConfig fessConfig;
86
87
88
89
90 protected DataSerializer dataSerializer;
91
92
93
94
95
96
97 protected abstract Extractor getExtractor(ResponseData responseData);
98
99 @Override
100 public ResultData transform(final ResponseData responseData) {
101 if (responseData == null || !responseData.hasResponseBody()) {
102 final String url = responseData != null ? responseData.getUrl() : "unknown";
103 throw new CrawlingAccessException("No response body for URL: " + url + ". Cannot transform empty response.");
104 }
105
106 final ResultData resultData = new ResultData();
107 resultData.setTransformerName(getName());
108 try {
109 resultData.setRawData(generateData(responseData));
110 resultData.setSerializer(dataSerializer::fromObjectToBinary);
111 } catch (final Exception e) {
112 throw new CrawlingAccessException("Could not serialize object", e);
113 }
114 resultData.setEncoding(fessConfig.getCrawlerCrawlingDataEncoding());
115
116 return resultData;
117 }
118
119
120
121
122
123
124 protected Map<String, Object> generateData(final ResponseData responseData) {
125 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
126 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
127 final Extractor extractor = getExtractor(responseData);
128 final String mimeType = responseData.getMimeType();
129 final StringBuilder contentMetaBuf = new StringBuilder(1000);
130 final Map<String, Object> dataMap = new HashMap<>();
131 final Map<String, Object> metaDataMap = new HashMap<>();
132 String content;
133 try (final InputStream in = responseData.getResponseBody()) {
134 final ExtractData extractData = getExtractData(extractor, in, createExtractParams(responseData, crawlingConfig));
135 content = extractData.getContent();
136 if (fessConfig.isCrawlerDocumentFileIgnoreEmptyContent() && StringUtil.isBlank(content)) {
137 return null;
138 }
139 if (getLogger().isDebugEnabled()) {
140 getLogger().debug("ExtractData: {}", extractData);
141 }
142
143 extractData.getKeySet().stream().filter(k -> extractData.getValues(k) != null).forEach(key -> {
144 final String[] values = extractData.getValues(key);
145 metaDataMap.put(key, values);
146
147
148 if (fessConfig.isCrawlerMetadataContentIncluded(key)) {
149 final String joinedValue = StringUtils.join(values, ' ');
150 if (StringUtil.isNotBlank(joinedValue)) {
151 if (contentMetaBuf.length() > 0) {
152 contentMetaBuf.append(' ');
153 }
154 contentMetaBuf.append(joinedValue.trim());
155 }
156 }
157
158 final Tuple3<String, String, String> mapping = fessConfig.getCrawlerMetadataNameMapping(key);
159 if (mapping != null) {
160 if (Constants.MAPPING_TYPE_ARRAY.equalsIgnoreCase(mapping.getValue2())) {
161 dataMap.put(mapping.getValue1(), values);
162 } else if (Constants.MAPPING_TYPE_STRING.equalsIgnoreCase(mapping.getValue2())) {
163 final String joinedValue = StringUtils.join(values, ' ');
164 dataMap.put(mapping.getValue1(), joinedValue.trim());
165 } else if (values.length == 1) {
166 try {
167 if (Constants.MAPPING_TYPE_LONG.equalsIgnoreCase(mapping.getValue2())) {
168 dataMap.put(mapping.getValue1(), Long.parseLong(values[0]));
169 } else if (Constants.MAPPING_TYPE_DOUBLE.equalsIgnoreCase(mapping.getValue2())) {
170 dataMap.put(mapping.getValue1(), Double.parseDouble(values[0]));
171 } else if (Constants.MAPPING_TYPE_DATE.equalsIgnoreCase(mapping.getValue2())
172 || Constants.MAPPING_TYPE_PDF_DATE.equalsIgnoreCase(mapping.getValue2())) {
173 final String dateFormat;
174 if (StringUtil.isNotBlank(mapping.getValue3())) {
175 dateFormat = mapping.getValue3();
176 } else if (Constants.MAPPING_TYPE_PDF_DATE.equalsIgnoreCase(mapping.getValue2())) {
177 dateFormat = Constants.MAPPING_TYPE_PDF_DATE;
178 } else {
179 dateFormat = Constants.DATE_OPTIONAL_TIME;
180 }
181 final Date dt = FessFunctions.parseDate(values[0], dateFormat);
182 if (dt != null) {
183 dataMap.put(mapping.getValue1(), FessFunctions.formatDate(dt));
184 } else {
185 logger.warn("Failed to parse date mapping: {}", mapping);
186 }
187 } else {
188 logger.warn("Unknown mapping type: {}={}", key, mapping);
189 }
190 } catch (final Exception e) {
191 logger.warn("Failed to parse value: {}", values[0], e);
192 }
193 }
194 }
195
196 });
197 } catch (final Exception e) {
198 final CrawlingAccessException rcae = new CrawlingAccessException("Could not get a text from " + responseData.getUrl(), e);
199 rcae.setLogLevel(CrawlingAccessException.WARN);
200 throw rcae;
201 }
202 if (content == null) {
203 content = StringUtil.EMPTY;
204 }
205 final String contentMeta = contentMetaBuf.toString().trim();
206
207 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
208 final String sessionId = crawlingInfoHelper.getCanonicalSessionId(responseData.getSessionId());
209 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
210 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
211 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
212 final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
213 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
214 String url = responseData.getUrl();
215 final String indexingTarget = crawlingConfig.getIndexingTarget(url);
216 url = pathMappingHelper.replaceUrl(sessionId, url);
217
218 final FieldConfigs fieldConfigs = new FieldConfigs(crawlingConfig.getConfigParameterMap(ConfigName.FIELD));
219
220 String urlEncoding;
221 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
222 if (urlQueue != null && urlQueue.getEncoding() != null) {
223 urlEncoding = urlQueue.getEncoding();
224 } else {
225 urlEncoding = responseData.getCharSet();
226 }
227
228
229 final String configId = crawlingConfig.getConfigId();
230 if (configId != null) {
231 putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
232 }
233
234 if (documentExpires != null) {
235 putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), documentExpires);
236 }
237
238 putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
239
240 final StringBuilder buf = new StringBuilder(content.length() + 1000);
241 if (fessConfig.isCrawlerDocumentFileAppendBodyContent()) {
242 buf.append(content);
243 }
244 if (fessConfig.isCrawlerDocumentFileAppendMetaContent()) {
245 if (buf.length() > 0) {
246 buf.append(' ');
247 }
248 buf.append(contentMeta);
249 }
250 final String fileName = getFileName(url, urlEncoding);
251 if (StringUtil.isNotBlank(fileName) && fessConfig.isCrawlerDocumentAppendFilename()) {
252 buf.append(' ').append(fileName);
253 }
254 final String bodyBase = buf.toString().trim();
255 responseData.addMetaData(Extractor.class.getSimpleName(), extractor);
256 final String body = documentHelper.getContent(crawlingConfig, responseData, bodyBase, dataMap);
257 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), body);
258 if ((fieldConfigs.getConfig(fessConfig.getIndexFieldCache())
259 .map(org.codelibs.fess.crawler.util.FieldConfigs.Config::isCache)
260 .orElse(false) || fessConfig.isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
261 if (responseData.getContentLength() > 0
262 && responseData.getContentLength() <= fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger().longValue()) {
263
264 final String cache = content.trim().replaceAll("[ \\t\\x0B\\f]+", " ");
265
266
267 putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), cache);
268 putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
269 }
270 }
271
272 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
273 documentHelper.getDigest(responseData, bodyBase, dataMap, fessConfig.getCrawlerDocumentFileMaxDigestLengthAsInteger()));
274
275 if (!hasTitle(dataMap)) {
276 final String titleField = fessConfig.getIndexFieldTitle();
277 dataMap.remove(titleField);
278 if (url.endsWith("/")) {
279 if (StringUtil.isNotBlank(content)) {
280 putResultDataBody(dataMap, titleField, documentHelper.getDigest(responseData, body, dataMap,
281 fessConfig.getCrawlerDocumentFileMaxTitleLengthAsInteger()));
282 } else {
283 putResultDataBody(dataMap, titleField, fessConfig.getCrawlerDocumentFileNoTitleLabel());
284 }
285 } else if (StringUtil.isBlank(fileName)) {
286 putResultDataBody(dataMap, titleField, decodeUrlAsName(url, url.startsWith("file:")));
287 } else {
288 putResultDataBody(dataMap, titleField, fileName);
289 }
290 }
291
292 putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHostOnFile(url));
293
294 putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSiteOnFile(url, urlEncoding));
295
296 if (StringUtil.isNotBlank(fileName)) {
297 putResultDataBody(dataMap, fessConfig.getIndexFieldFilename(), fileName);
298 }
299
300 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
301
302 final Date now = systemHelper.getCurrentTime();
303 putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), now);
304
305 putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), StringUtil.EMPTY);
306
307 putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
308 if (fileTypeHelper != null) {
309
310 putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
311 }
312
313 putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
314
315 final Date lastModified = getLastModified(dataMap, responseData);
316 if (lastModified != null) {
317 dataMap.put(fessConfig.getIndexFieldLastModified(), lastModified);
318
319 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), lastModified);
320 } else {
321
322 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), now);
323 }
324
325 putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
326
327 putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
328
329 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
330 putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeHelper.getMatchedLabelValueSet(url));
331
332 final List<String> roleTypeList = getRoleTypes(responseData);
333 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
334 putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
335
336 putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
337 stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
338
339
340 if (StringUtil.isNotBlank(fessConfig.getCrawlerDocumentFileDefaultLang())) {
341 putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), fessConfig.getCrawlerDocumentFileDefaultLang());
342 }
343
344 putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
345
346 String parentUrl = responseData.getParentUrl();
347 if (StringUtil.isNotBlank(parentUrl)) {
348 parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
349 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
350 putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
351 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
352 }
353
354 putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), responseData.getUrl());
355
356
357 final String scriptType = crawlingConfig.getScriptType();
358 final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);
359 final Map<String, String> metaConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.META);
360 for (final Map.Entry<String, String> entry : metaConfigMap.entrySet()) {
361 final String key = entry.getKey();
362 final String[] values = entry.getValue().split(",");
363 for (final String value : values) {
364 putResultDataWithTemplate(dataMap, key, metaDataMap.get(value), scriptConfigMap.get(key), scriptType);
365 }
366 }
367 final Map<String, String> valueConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.VALUE);
368 for (final Map.Entry<String, String> entry : valueConfigMap.entrySet()) {
369 final String key = entry.getKey();
370 putResultDataWithTemplate(dataMap, key, entry.getValue(), scriptConfigMap.get(key), scriptType);
371 }
372
373 return processFieldConfigs(dataMap, fieldConfigs);
374 }
375
376
377
378
379
380
381
382 protected Date getLastModified(final Map<String, Object> dataMap, final ResponseData responseData) {
383 final Object lastModifiedObj = dataMap.get(fessConfig.getIndexFieldLastModified());
384 if (lastModifiedObj instanceof Date) {
385 return (Date) lastModifiedObj;
386 }
387 if (lastModifiedObj instanceof String) {
388 final Date lastModified = FessFunctions.parseDate(lastModifiedObj.toString());
389 if (lastModified != null) {
390 return lastModified;
391 }
392 } else if (lastModifiedObj instanceof final String[] lastModifieds && lastModifieds.length > 0) {
393 final Date lastModified = FessFunctions.parseDate(lastModifieds[0]);
394 if (lastModified != null) {
395 return lastModified;
396 }
397 }
398
399 return responseData.getLastModified();
400 }
401
402
403
404
405
406
407 protected boolean hasTitle(final Map<String, Object> dataMap) {
408 final Object titleObj = dataMap.get(fessConfig.getIndexFieldTitle());
409 if (titleObj != null) {
410 if (titleObj instanceof String[]) {
411 return stream((String[]) titleObj).get(stream -> stream.anyMatch(StringUtil::isNotBlank));
412 }
413 return StringUtil.isNotBlank(titleObj.toString());
414 }
415 return false;
416 }
417
418
419
420
421
422
423
424 protected Map<String, String> createExtractParams(final ResponseData responseData, final CrawlingConfig crawlingConfig) {
425 final Map<String, String> params = new HashMap<>(crawlingConfig.getConfigParameterMap(ConfigName.CONFIG));
426 params.put(ExtractData.RESOURCE_NAME_KEY, getResourceName(responseData));
427 params.put(ExtractData.CONTENT_TYPE, responseData.getMimeType());
428 params.put(ExtractData.CONTENT_ENCODING, responseData.getCharSet());
429 params.put(ExtractData.URL, responseData.getUrl());
430 final Map<String, String> configParam = crawlingConfig.getConfigParameterMap(ConfigName.CONFIG);
431 if (configParam != null) {
432 final String keepOriginalBody = configParam.get(Config.KEEP_ORIGINAL_BODY);
433 if (StringUtil.isNotBlank(keepOriginalBody)) {
434 params.put(TikaExtractor.NORMALIZE_TEXT,
435 Constants.TRUE.equalsIgnoreCase(keepOriginalBody) ? Constants.FALSE : Constants.TRUE);
436 }
437 }
438 return params;
439 }
440
441
442
443
444
445
446
447
448 protected ExtractData getExtractData(final Extractor extractor, final InputStream in, final Map<String, String> params) {
449 try {
450 return extractor.getText(in, params);
451 } catch (final RuntimeException e) {
452 if (!fessConfig.isCrawlerIgnoreContentException()) {
453 throw e;
454 }
455 if (logger.isDebugEnabled()) {
456 logger.debug("Could not get a text.", e);
457 }
458 }
459 return new ExtractData();
460 }
461
462
463
464
465
466
467 protected String getResourceName(final ResponseData responseData) {
468 String name = responseData.getUrl();
469 final String enc = responseData.getCharSet();
470
471 if (name == null || enc == null) {
472 return null;
473 }
474
475 name = name.replaceAll("/+$", StringUtil.EMPTY);
476 final int idx = name.lastIndexOf('/');
477 if (idx >= 0) {
478 name = name.substring(idx + 1);
479 }
480 try {
481 return URLDecoder.decode(name, enc);
482 } catch (final Exception e) {
483 return name;
484 }
485 }
486
487
488
489
490
491
492 protected String getHostOnFile(final String url) {
493 if (StringUtil.isBlank(url)) {
494 return StringUtil.EMPTY;
495 }
496
497 if (url.startsWith("file:////")) {
498 final String value = decodeUrlAsName(url.substring(9), true);
499 final int pos = value.indexOf('/');
500 if (pos > 0) {
501 return value.substring(0, pos);
502 }
503 if (pos == -1) {
504 return value;
505 }
506 return "localhost";
507 }
508 if (url.startsWith("file:")) {
509 return "localhost";
510 }
511
512 return getHost(url);
513 }
514
515
516
517
518
519
520 protected List<String> getRoleTypes(final ResponseData responseData) {
521 final List<String> roleTypeList = new ArrayList<>();
522 final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
523
524 roleTypeList.addAll(permissionHelper.getSmbRoleTypeList(responseData));
525 roleTypeList.addAll(permissionHelper.getFileRoleTypeList(responseData));
526 roleTypeList.addAll(permissionHelper.getFtpRoleTypeList(responseData));
527
528 return roleTypeList;
529 }
530
531
532
533
534
535
536
537 protected String getSiteOnFile(final String url, final String encoding) {
538 if (StringUtil.isBlank(url)) {
539 return StringUtil.EMPTY;
540 }
541
542 if (url.startsWith("file:////")) {
543 final String value = decodeUrlAsName(url.substring(9), true);
544 return abbreviateSite("\\\\" + value.replace('/', '\\'));
545 }
546 if (url.startsWith("file:")) {
547 final String value = decodeUrlAsName(url.substring(5), true);
548 if (value.length() > 2 && value.charAt(2) == ':') {
549
550 return abbreviateSite(value.substring(1).replace('/', '\\'));
551 }
552
553 return abbreviateSite(value);
554 }
555 if (url.startsWith("smb:") || url.startsWith("smb1:")) {
556 final String value = url.replaceFirst("^smb.?:/+", StringUtil.EMPTY);
557 return abbreviateSite("\\\\" + value.replace('/', '\\'));
558 }
559
560 return getSite(url, encoding);
561 }
562
563 @Override
564 public Object getData(final AccessResultData<?> accessResultData) {
565 final byte[] data = accessResultData.getData();
566 if (data != null) {
567 try {
568 return dataSerializer.fromBinaryToObject(data);
569 } catch (final Exception e) {
570 throw new CrawlerSystemException("Could not create an instanced from bytes.", e);
571 }
572 }
573 return new HashMap<String, Object>();
574 }
575
576
577
578
579
580
581 public void addMetaContentMapping(final String metaname, final String dynamicField) {
582 if (metaContentMapping == null) {
583 metaContentMapping = new HashMap<>();
584 }
585 metaContentMapping.put(metaname, dynamicField);
586 }
587
588 }