1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedInputStream;
21 import java.net.MalformedURLException;
22 import java.net.URL;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.Collections;
26 import java.util.Date;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.LinkedHashMap;
30 import java.util.LinkedHashSet;
31 import java.util.List;
32 import java.util.Locale;
33 import java.util.Map;
34 import java.util.Set;
35 import java.util.function.UnaryOperator;
36 import java.util.stream.Collectors;
37 import java.util.stream.Stream;
38
39 import javax.xml.xpath.XPathEvaluationResult;
40 import javax.xml.xpath.XPathExpressionException;
41 import javax.xml.xpath.XPathNodes;
42
43 import org.apache.logging.log4j.LogManager;
44 import org.apache.logging.log4j.Logger;
45 import org.codelibs.core.io.InputStreamUtil;
46 import org.codelibs.core.lang.StringUtil;
47 import org.codelibs.core.misc.Pair;
48 import org.codelibs.core.misc.ValueHolder;
49 import org.codelibs.fess.Constants;
50 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
51 import org.codelibs.fess.crawler.entity.AccessResultData;
52 import org.codelibs.fess.crawler.entity.RequestData;
53 import org.codelibs.fess.crawler.entity.ResponseData;
54 import org.codelibs.fess.crawler.entity.ResultData;
55 import org.codelibs.fess.crawler.entity.UrlQueue;
56 import org.codelibs.fess.crawler.exception.ChildUrlsException;
57 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
58 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
59 import org.codelibs.fess.crawler.serializer.DataSerializer;
60 import org.codelibs.fess.crawler.transformer.impl.XpathTransformer;
61 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
62 import org.codelibs.fess.crawler.util.FieldConfigs;
63 import org.codelibs.fess.helper.CrawlingConfigHelper;
64 import org.codelibs.fess.helper.CrawlingInfoHelper;
65 import org.codelibs.fess.helper.DocumentHelper;
66 import org.codelibs.fess.helper.DuplicateHostHelper;
67 import org.codelibs.fess.helper.FileTypeHelper;
68 import org.codelibs.fess.helper.LabelTypeHelper;
69 import org.codelibs.fess.helper.PathMappingHelper;
70 import org.codelibs.fess.helper.SystemHelper;
71 import org.codelibs.fess.mylasta.direction.FessConfig;
72 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig;
73 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.ConfigName;
74 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.Param.Config;
75 import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig.Param.XPath;
76 import org.codelibs.fess.util.ComponentUtil;
77 import org.codelibs.fess.util.PrunedTag;
78 import org.codelibs.nekohtml.parsers.DOMParser;
79 import org.w3c.dom.Document;
80 import org.w3c.dom.NamedNodeMap;
81 import org.w3c.dom.Node;
82 import org.w3c.dom.NodeList;
83 import org.xml.sax.InputSource;
84
85 import jakarta.annotation.PostConstruct;
86
87
88
89
90
91
92 public class FessXpathTransformer extends XpathTransformer implements FessTransformer {
93
94
95 private static final Logger logger = LogManager.getLogger(FessXpathTransformer.class);
96
97
98 private static final String X_ROBOTS_TAG = "X-Robots-Tag";
99
100
101 private static final String META_NAME_THUMBNAIL_CONTENT = "//META[@name=\"thumbnail\" or @name=\"THUMBNAIL\"]/@content";
102
103
104 private static final String META_PROPERTY_OGIMAGE_CONTENT = "//META[@property=\"og:image\"]/@content";
105
106
107 private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";
108
109
110 private static final String ROBOTS_TAG_NONE = "none";
111
112
113 private static final String ROBOTS_TAG_NOINDEX = "noindex";
114
115
116 private static final String ROBOTS_TAG_NOFOLLOW = "nofollow";
117
118
119 private static final int UTF8_BOM_SIZE = 3;
120
121
122 public boolean prunedContent = true;
123
124
125 protected Map<String, String> convertUrlMap = new LinkedHashMap<>();
126
127
128 protected FessConfig fessConfig;
129
130
131 protected DataSerializer dataSerializer;
132
133
134 protected boolean useGoogleOffOn = true;
135
136
137 protected Map<String, Boolean> fieldPrunedRuleMap = new HashMap<>();
138
139
140 protected Map<String, PrunedTag[]> prunedTagsCache = new HashMap<>();
141
142
143
144
145 public FessXpathTransformer() {
146 super();
147 }
148
149
150
151
152
153 @PostConstruct
154 public void init() {
155 if (logger.isDebugEnabled()) {
156 logger.debug("Initializing {}", this.getClass().getSimpleName());
157 }
158 fessConfig = ComponentUtil.getFessConfig();
159 dataSerializer = ComponentUtil.getComponent("dataSerializer");
160 }
161
162
163
164
165
166
167 @Override
168 public FessConfig getFessConfig() {
169 return fessConfig;
170 }
171
172
173
174
175
176
177 @Override
178 public Logger getLogger() {
179 return logger;
180 }
181
182
183
184
185
186
187
188
189 @Override
190 protected void storeData(final ResponseData responseData, final ResultData resultData) {
191 final DOMParser parser = getDomParser();
192 try (final BufferedInputStream bis = new BufferedInputStream(responseData.getResponseBody())) {
193 final byte[] bomBytes = new byte[UTF8_BOM_SIZE];
194 bis.mark(UTF8_BOM_SIZE);
195 final int size = bis.read(bomBytes);
196 if (size < 3 || !isUtf8BomBytes(bomBytes)) {
197 bis.reset();
198 }
199 final InputSource is = new InputSource(bis);
200 if (responseData.getCharSet() != null) {
201 is.setEncoding(responseData.getCharSet());
202 }
203 parser.parse(is);
204 } catch (final Exception e) {
205 throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);
206 }
207
208 final Document document = parser.getDocument();
209
210 processMetaRobots(responseData, resultData, document);
211 processXRobotsTag(responseData, resultData);
212
213 Map<String, Object> dataMap = new LinkedHashMap<>();
214 for (final Map.Entry<String, String> entry : fieldRuleMap.entrySet()) {
215 final String path = entry.getValue();
216 try {
217 final XPathEvaluationResult<?> xObj = getXPathAPI().eval(document, path);
218 switch (xObj.type()) {
219 case BOOLEAN:
220 final Boolean b = (Boolean) xObj.value();
221 putResultDataBody(dataMap, entry.getKey(), b.toString());
222 break;
223 case NUMBER:
224 final Number d = (Number) xObj.value();
225 putResultDataBody(dataMap, entry.getKey(), d.toString());
226 break;
227 case STRING:
228 final String str = (String) xObj.value();
229 putResultDataBody(dataMap, entry.getKey(), str);
230 break;
231 default:
232 final Boolean isPruned = fieldPrunedRuleMap.get(entry.getKey());
233 Node value = getXPathAPI().selectSingleNode(document, entry.getValue());
234 if (value != null && isPruned != null && isPruned.booleanValue()) {
235 value = pruneNode(value, getCrawlingConfig(responseData));
236 }
237 putResultDataBody(dataMap, entry.getKey(), value != null ? value.getTextContent() : null);
238 break;
239 }
240 } catch (final XPathExpressionException e) {
241 logger.warn("Could not parse a value of {}:{}", entry.getKey(), entry.getValue(), e);
242 }
243 }
244
245 dataMap = processAdditionalData(dataMap, responseData, document);
246 normalizeData(responseData, dataMap);
247
248 try {
249 resultData.setRawData(dataMap);
250 resultData.setSerializer(dataSerializer::fromObjectToBinary);
251 } catch (final Exception e) {
252 throw new CrawlingAccessException("Could not serialize object: " + responseData.getUrl(), e);
253 }
254 resultData.setEncoding(charsetName);
255 }
256
257
258
259
260
261
262
263 protected void normalizeData(final ResponseData responseData, final Map<String, Object> dataMap) {
264 final Object titleObj = dataMap.get(fessConfig.getIndexFieldTitle());
265 if (titleObj != null) {
266 dataMap.put(fessConfig.getIndexFieldTitle(),
267 ComponentUtil.getDocumentHelper().getTitle(responseData, titleObj.toString(), dataMap));
268 }
269 }
270
271
272
273
274
275
276
277
278
279 protected void processMetaRobots(final ResponseData responseData, final ResultData resultData, final Document document) {
280 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
281 final String ignore = configMap.get(Config.IGNORE_ROBOTS_TAGS);
282 if (ignore == null) {
283 if (fessConfig.isCrawlerIgnoreRobotsTags()) {
284 return;
285 }
286 } else if (Boolean.parseBoolean(ignore)) {
287 return;
288 }
289
290
291 try {
292 final Node value = getXPathAPI().selectSingleNode(document, META_NAME_ROBOTS_CONTENT);
293 if (value != null) {
294 boolean noindex = false;
295 boolean nofollow = false;
296 final String content = value.getTextContent().toLowerCase(Locale.ROOT);
297 if (content.contains(ROBOTS_TAG_NONE)) {
298 noindex = true;
299 nofollow = true;
300 } else {
301 if (content.contains(ROBOTS_TAG_NOINDEX)) {
302 noindex = true;
303 }
304 if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
305 nofollow = true;
306 }
307 }
308 if (noindex && nofollow) {
309 logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
310 throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
311 }
312 if (noindex) {
313 logger.info("META(robots=noindex): {}", responseData.getUrl());
314 storeChildUrls(responseData, resultData);
315 throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
316 }
317 if (nofollow) {
318 logger.info("META(robots=nofollow): {}", responseData.getUrl());
319 responseData.setNoFollow(true);
320 }
321 }
322 } catch (final XPathExpressionException e) {
323 logger.warn("Could not parse a value of {}", META_NAME_ROBOTS_CONTENT, e);
324 }
325
326 }
327
328
329
330
331
332
333
334
335 protected void processXRobotsTag(final ResponseData responseData, final ResultData resultData) {
336 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
337 final String ignore = configMap.get(Config.IGNORE_ROBOTS_TAGS);
338 if (ignore == null) {
339 if (fessConfig.isCrawlerIgnoreRobotsTags()) {
340 return;
341 }
342 } else if (Boolean.parseBoolean(ignore)) {
343 return;
344 }
345
346
347 responseData.getMetaDataMap()
348 .entrySet()
349 .stream()
350 .filter(e -> X_ROBOTS_TAG.equalsIgnoreCase(e.getKey()) && e.getValue() != null)
351 .forEach(e -> {
352 boolean noindex = false;
353 boolean nofollow = false;
354 final String value = e.getValue().toString().toLowerCase(Locale.ROOT);
355 if (value.contains(ROBOTS_TAG_NONE)) {
356 noindex = true;
357 nofollow = true;
358 } else {
359 if (value.contains(ROBOTS_TAG_NOINDEX)) {
360 noindex = true;
361 }
362 if (value.contains(ROBOTS_TAG_NOFOLLOW)) {
363 nofollow = true;
364 }
365 }
366 if (noindex && nofollow) {
367 logger.info("HEADER(robots=noindex,nofollow): {}", responseData.getUrl());
368 throw new ChildUrlsException(Collections.emptySet(), "#processXRobotsTag");
369 }
370 if (noindex) {
371 logger.info("HEADER(robots=noindex): {}", responseData.getUrl());
372 storeChildUrls(responseData, resultData);
373 throw new ChildUrlsException(resultData.getChildUrlSet(), "#processXRobotsTag");
374 }
375 if (nofollow) {
376 logger.info("HEADER(robots=nofollow): {}", responseData.getUrl());
377 responseData.setNoFollow(true);
378 }
379 });
380 }
381
382
383
384
385
386
387
388
389 protected Map<String, String> getConfigPrameterMap(final ResponseData responseData, final ConfigName config) {
390 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
391 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
392 return crawlingConfig.getConfigParameterMap(config);
393 }
394
395
396
397
398
399
400
401 protected boolean isValidUrl(final String urlStr) {
402 if (StringUtil.isBlank(urlStr)) {
403 return false;
404 }
405 final String value;
406 if (urlStr.startsWith("://")) {
407 value = "http" + urlStr;
408 } else if (urlStr.startsWith("//")) {
409 value = "http:" + urlStr;
410 } else {
411 value = urlStr;
412 }
413 try {
414 final URL url = new URL(value);
415 final String host = url.getHost();
416 if (StringUtil.isBlank(host) || "http".equalsIgnoreCase(host) || "https".equalsIgnoreCase(host)) {
417 return false;
418 }
419 } catch (final MalformedURLException e) {
420 return false;
421 }
422 return true;
423 }
424
425
426
427
428
429
430
431
432
433 protected boolean isValidCanonicalUrl(final String url, final String canonicalUrl) {
434 if (url.startsWith("https:") && canonicalUrl.startsWith("http:")) {
435 if (logger.isDebugEnabled()) {
436 logger.debug("Invalid Canonical Url(https->http): {} -> {}", url, canonicalUrl);
437 }
438 return false;
439 }
440 return true;
441 }
442
443
444
445
446
447
448
449
450
451 protected Map<String, Object> processAdditionalData(final Map<String, Object> dataMap, final ResponseData responseData,
452 final Document document) {
453
454 final String canonicalUrl = getCanonicalUrl(responseData, document);
455 if (canonicalUrl != null && !canonicalUrl.equalsIgnoreCase(responseData.getUrl()) && isValidUrl(canonicalUrl)
456 && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {
457 final Set<RequestData> childUrlSet = new HashSet<>();
458 childUrlSet.add(RequestDataBuilder.newRequestData().get().url(canonicalUrl).build());
459 logger.info("Canonical URL redirect: from={}, to={}", responseData.getUrl(), canonicalUrl);
460 throw new ChildUrlsException(childUrlSet, this.getClass().getName() + "#putAdditionalData");
461 }
462
463 final FessConfig fessConfig = ComponentUtil.getFessConfig();
464 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
465 final String sessionId = crawlingInfoHelper.getCanonicalSessionId(responseData.getSessionId());
466 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
467 final CrawlingConfig crawlingConfig = getCrawlingConfig(responseData);
468 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
469 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
470 final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
471 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
472 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
473 String url = responseData.getUrl();
474 final String indexingTarget = crawlingConfig.getIndexingTarget(url);
475 url = pathMappingHelper.replaceUrl(sessionId, url);
476 final String mimeType = responseData.getMimeType();
477
478 final FieldConfigs fieldConfigs = new FieldConfigs(crawlingConfig.getConfigParameterMap(ConfigName.FIELD));
479 final Map<String, String> xpathConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.XPATH);
480
481 String urlEncoding;
482 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
483 if (urlQueue != null && urlQueue.getEncoding() != null) {
484 urlEncoding = urlQueue.getEncoding();
485 } else {
486 urlEncoding = responseData.getCharSet();
487 }
488
489
490 final String configId = crawlingConfig.getConfigId();
491 if (configId != null) {
492 putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
493 }
494
495 if (documentExpires != null) {
496 putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), documentExpires);
497 }
498
499 final String lang = systemHelper.normalizeHtmlLang(
500 getSingleNodeValue(document, getLangXpath(fessConfig, xpathConfigMap), node -> pruneNode(node, crawlingConfig)));
501 if (lang != null) {
502 putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), lang);
503 }
504
505
506 final String body = getSingleNodeValue(document, getContentXpath(fessConfig, xpathConfigMap),
507 prunedContent ? node -> pruneNode(node, crawlingConfig) : node -> node);
508 final String fileName = getFileName(url, urlEncoding);
509 putResultDataContent(dataMap, responseData, fessConfig, crawlingConfig, documentHelper, body, fileName);
510 if ((fieldConfigs.getConfig(fessConfig.getIndexFieldCache())
511 .map(org.codelibs.fess.crawler.util.FieldConfigs.Config::isCache)
512 .orElse(false) || fessConfig.isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
513 if (responseData.getContentLength() > 0
514 && responseData.getContentLength() <= fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger().longValue()) {
515 String charSet = responseData.getCharSet();
516 if (charSet == null) {
517 charSet = Constants.UTF_8;
518 }
519 try (final BufferedInputStream is = new BufferedInputStream(responseData.getResponseBody())) {
520
521 putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), new String(InputStreamUtil.getBytes(is), charSet));
522 putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
523 } catch (final Exception e) {
524 logger.warn("Failed to write cache: sessionId={}, responseData={}", sessionId, responseData, e);
525 }
526 } else {
527 logger.debug("Content size is too large({} > {}): {}", responseData.getContentLength(),
528 fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger(), responseData.getUrl());
529 }
530 }
531
532 final String digest = getSingleNodeValue(document, getDigestXpath(fessConfig, xpathConfigMap), node -> node);
533 if (StringUtil.isNotBlank(digest)) {
534 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(), digest);
535 } else {
536 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
537 documentHelper.getDigest(responseData, body, dataMap, fessConfig.getCrawlerDocumentHtmlMaxDigestLengthAsInteger()));
538 }
539
540 putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
541
542 putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHost(url));
543
544 putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSite(url, urlEncoding));
545
546 if (StringUtil.isNotBlank(fileName)) {
547 putResultDataBody(dataMap, fessConfig.getIndexFieldFilename(), fileName);
548 }
549
550 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
551
552 final Date now = systemHelper.getCurrentTime();
553 putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), now);
554
555 putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), getAnchorList(document, responseData));
556
557 putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
558 if (fileTypeHelper != null) {
559
560 putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
561 }
562
563 putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
564
565 final Date lastModified = responseData.getLastModified();
566 if (lastModified != null) {
567 putResultDataBody(dataMap, fessConfig.getIndexFieldLastModified(), lastModified);
568
569 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), lastModified);
570 } else {
571
572 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), now);
573 }
574
575 putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
576
577 putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
578
579 putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeHelper.getMatchedLabelValueSet(url));
580
581 final List<String> roleTypeList = new ArrayList<>();
582 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
583 putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
584
585 putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
586 stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
587
588 putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
589
590 String parentUrl = responseData.getParentUrl();
591 if (StringUtil.isNotBlank(parentUrl)) {
592 parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
593 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
594 putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
595 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
596 }
597
598 final String thumbnailUrl = getThumbnailUrl(responseData, document);
599 if (StringUtil.isNotBlank(thumbnailUrl)) {
600 putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), thumbnailUrl);
601 }
602
603
604 final String scriptType = crawlingConfig.getScriptType();
605 final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);
606 xpathConfigMap.entrySet().stream().filter(e -> !e.getKey().startsWith("default.")).forEach(e -> {
607 final String key = e.getKey();
608 final String value = getSingleNodeValue(document, e.getValue(), node -> pruneNode(node, crawlingConfig));
609 putResultDataWithTemplate(dataMap, key, value, scriptConfigMap.get(key), scriptType);
610 });
611 crawlingConfig.getConfigParameterMap(ConfigName.VALUE).entrySet().stream().forEach(e -> {
612 final String key = e.getKey();
613 final String value = e.getValue();
614 putResultDataWithTemplate(dataMap, key, value, scriptConfigMap.get(key), scriptType);
615 });
616
617 return processFieldConfigs(dataMap, fieldConfigs);
618 }
619
620
621
622
623
624
625
626
627
628
629
630
631 protected void putResultDataContent(final Map<String, Object> dataMap, final ResponseData responseData, final FessConfig fessConfig,
632 final CrawlingConfig crawlingConfig, final DocumentHelper documentHelper, final String body, final String fileName) {
633 final String content = documentHelper.getContent(crawlingConfig, responseData, body, dataMap);
634 if (StringUtil.isNotBlank(fileName) && fessConfig.isCrawlerDocumentAppendFilename()) {
635 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), content + " " + fileName);
636 } else {
637 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), content);
638 }
639 }
640
641
642
643
644
645
646
647 protected CrawlingConfig getCrawlingConfig(final ResponseData responseData) {
648 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
649 return crawlingConfigHelper.get(responseData.getSessionId());
650 }
651
652
653
654
655
656
657
658
659 protected String getLangXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
660 final String xpath = xpathConfigMap.get(XPath.DEFAULT_LANG);
661 if (StringUtil.isNotBlank(xpath)) {
662 return xpath;
663 }
664 return fessConfig.getCrawlerDocumentHtmlLangXpath();
665 }
666
667
668
669
670
671
672
673
674 protected String getContentXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
675 final String xpath = xpathConfigMap.get(XPath.DEFAULT_CONTENT);
676 if (StringUtil.isNotBlank(xpath)) {
677 return xpath;
678 }
679 return fessConfig.getCrawlerDocumentHtmlContentXpath();
680 }
681
682
683
684
685
686
687
688
689 protected String getDigestXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
690 final String xpath = xpathConfigMap.get(XPath.DEFAULT_DIGEST);
691 if (StringUtil.isNotBlank(xpath)) {
692 return xpath;
693 }
694 return fessConfig.getCrawlerDocumentHtmlDigestXpath();
695 }
696
697
698
699
700
701
702
703
704 protected String getCanonicalUrl(final ResponseData responseData, final Document document) {
705 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
706 String xpath = configMap.get(Config.HTML_CANONICAL_XPATH);
707 if (xpath == null) {
708 xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath();
709 }
710 if (StringUtil.isBlank(xpath)) {
711 return null;
712 }
713 final String canonicalUrl = getSingleNodeValue(document, xpath, node -> node);
714 if (StringUtil.isBlank(canonicalUrl)) {
715 return null;
716 }
717 return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
718 }
719
720
721
722
723
724
725
726
727 protected String normalizeCanonicalUrl(final String baseUrl, final String canonicalUrl) {
728 try {
729 final URL u = new URL(baseUrl);
730 final String resolveTarget = canonicalUrl.startsWith(":") ? u.getProtocol() + canonicalUrl : canonicalUrl;
731 return new URL(u, resolveTarget).toString();
732 } catch (final MalformedURLException e) {
733 logger.warn("Invalid canonical URL: baseUrl={}, canonicalUrl={}", baseUrl, canonicalUrl, e);
734 }
735 return null;
736 }
737
738
739
740
741
742
743
744 protected String removeCommentTag(final String content) {
745 if (content == null) {
746 return StringUtil.EMPTY;
747 }
748 String value = content;
749 int pos = value.indexOf("<!--");
750 while (pos >= 0) {
751 final int lastPos = value.indexOf("-->", pos);
752 if (lastPos < 0) {
753 break;
754 }
755 if (pos == 0) {
756 value = " " + value.substring(lastPos + 3);
757 } else {
758 value = value.substring(0, pos) + " " + value.substring(lastPos + 3);
759 }
760 pos = value.indexOf("<!--");
761 }
762 return value;
763 }
764
765
766
767
768
769
770
771
772
773 protected String getSingleNodeValue(final Document document, final String xpath, final UnaryOperator<Node> pruneFunc) {
774 StringBuilder buf = null;
775 XPathNodes list = null;
776 try {
777 list = getXPathAPI().selectNodeList(document, xpath);
778 for (int i = 0; i < list.size(); i++) {
779 if (buf == null) {
780 buf = new StringBuilder(1000);
781 }
782 Node node = list.get(i).cloneNode(true);
783 if (useGoogleOffOn) {
784 node = processGoogleOffOn(node, new ValueHolder<>(true));
785 }
786 node = pruneFunc.apply(node);
787 parseTextContent(node, buf);
788 }
789 } catch (final Exception e) {
790 logger.warn("Could not parse a value of {}", xpath);
791 }
792 if (buf == null) {
793 return null;
794 }
795 return buf.toString().trim();
796 }
797
798
799
800
801
802
803
804 protected void parseTextContent(final Node node, final StringBuilder buf) {
805 if (node.hasChildNodes()) {
806 final NodeList nodeList = node.getChildNodes();
807 for (int i = 0; i < nodeList.getLength(); i++) {
808 final Node childNode = nodeList.item(i);
809 parseTextContent(childNode, buf);
810 }
811 } else if (node.getNodeType() == Node.TEXT_NODE) {
812 final String value = node.getTextContent();
813 if (value != null) {
814 final String content = value.trim();
815 if (content.length() > 0) {
816 buf.append(' ').append(content);
817 }
818 }
819 }
820 }
821
822
823
824
825
826
827
828
829 protected Node processGoogleOffOn(final Node node, final ValueHolder<Boolean> flag) {
830 final NodeList nodeList = node.getChildNodes();
831 List<Node> removedNodeList = null;
832 for (int i = 0; i < nodeList.getLength(); i++) {
833 final Node childNode = nodeList.item(i);
834 if (childNode.getNodeType() == Node.COMMENT_NODE) {
835 final String comment = childNode.getNodeValue().trim();
836 if (comment.startsWith("googleoff:")) {
837 flag.setValue(false);
838 } else if (comment.startsWith("googleon:")) {
839 flag.setValue(true);
840 }
841 }
842
843 if (!flag.getValue() && childNode.getNodeType() == Node.TEXT_NODE) {
844 if (removedNodeList == null) {
845 removedNodeList = new ArrayList<>();
846 }
847 removedNodeList.add(childNode);
848 } else {
849 processGoogleOffOn(childNode, flag);
850 }
851 }
852
853 if (removedNodeList != null) {
854 removedNodeList.stream().forEach(n -> node.removeChild(n));
855 }
856
857 return node;
858 }
859
860
861
862
863
864
865
866
867 protected Node pruneNode(final Node node, final CrawlingConfig crawlingConfig) {
868 PrunedTag[] prunedTags = null;
869 if (crawlingConfig != null) {
870 final String configId = crawlingConfig.getConfigId();
871 prunedTags = prunedTagsCache.get(configId);
872 if (prunedTags == null) {
873 final Map<String, String> configMap = crawlingConfig.getConfigParameterMap(ConfigName.CONFIG);
874 final String value = configMap.get(CrawlingConfig.Param.Config.HTML_PRUNED_TAGS);
875 if (StringUtil.isNotBlank(value)) {
876 prunedTags = PrunedTag.parse(value);
877 }
878 if (prunedTags == null) {
879 prunedTags = fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray();
880 }
881 prunedTagsCache.put(configId, prunedTags);
882 }
883 }
884 if (prunedTags == null) {
885 prunedTags = fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray();
886 }
887 return pruneNodeByTags(node, prunedTags);
888 }
889
890
891
892
893
894
895
896
897 protected Node pruneNodeByTags(final Node node, final PrunedTag[] prunedTags) {
898 final NodeList nodeList = node.getChildNodes();
899 final List<Node> childNodeList = new ArrayList<>();
900 final List<Node> removedNodeList = new ArrayList<>();
901 for (int i = 0; i < nodeList.getLength(); i++) {
902 final Node childNode = nodeList.item(i);
903 if (isPrunedTag(childNode, prunedTags)) {
904 removedNodeList.add(childNode);
905 } else {
906 childNodeList.add(childNode);
907 }
908 }
909
910 for (final Node childNode : removedNodeList) {
911 node.removeChild(childNode);
912 }
913
914 for (final Node childNode : childNodeList) {
915 pruneNodeByTags(childNode, prunedTags);
916 }
917
918 return node;
919 }
920
921
922
923
924
925
926
927
928 protected boolean isPrunedTag(final Node node, final PrunedTag[] prunedTags) {
929 for (final PrunedTag prunedTag : prunedTags) {
930 if (prunedTag.matches(node)) {
931 return true;
932 }
933 }
934 return false;
935 }
936
937
938
939
940
941
942
943
944 protected String getMultipleNodeValue(final Document document, final String xpath) {
945 XPathNodes nodeList = null;
946 final StringBuilder buf = new StringBuilder(100);
947 try {
948 nodeList = getXPathAPI().selectNodeList(document, xpath);
949 for (int i = 0; i < nodeList.size(); i++) {
950 final Node node = nodeList.get(i);
951 buf.append(node.getTextContent());
952 buf.append("\n");
953 }
954 } catch (final Exception e) {
955 logger.warn("Could not parse a value of {}", xpath, e);
956 }
957 return buf.toString().trim();
958 }
959
960
961
962
963
964
965
966 protected String replaceDuplicateHost(final String url) {
967 try {
968
969 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
970 return duplicateHostHelper.convert(url);
971 } catch (final Exception e) {
972 return url;
973 }
974 }
975
976
977
978
979
980
981
982
983 protected List<String> getAnchorList(final Document document, final ResponseData responseData) {
984 List<RequestData> anchorList = new ArrayList<>();
985 final String baseHref = getBaseHref(document);
986 try {
987 final URL url = getBaseUrl(responseData.getUrl(), baseHref);
988 for (final Map.Entry<String, String> entry : childUrlRuleMap.entrySet()) {
989 for (final String u : getUrlFromTagAttribute(url, document, entry.getKey(), entry.getValue(), responseData.getCharSet())) {
990 anchorList.add(RequestDataBuilder.newRequestData().get().url(u).build());
991 }
992 }
993 anchorList = convertChildUrlList(anchorList);
994 } catch (final Exception e) {
995 logger.warn("Could not parse anchor tags.", e);
996 }
997
998 final Set<String> urlSet = new LinkedHashSet<>(anchorList.size());
999 for (final RequestData requestData : anchorList) {
1000 urlSet.add(requestData.getUrl());
1001 }
1002 return new ArrayList<>(urlSet);
1003 }
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013 protected URL getBaseUrl(final String currentUrl, final String baseHref) throws MalformedURLException {
1014 if (baseHref != null) {
1015 return getURL(currentUrl, baseHref);
1016 }
1017 return new URL(currentUrl);
1018 }
1019
1020
1021
1022
1023
1024
1025
1026
1027 @Override
1028 protected Stream<Pair<String, String>> getChildUrlRules(final ResponseData responseData, final ResultData resultData) {
1029 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
1030 final String ruleString = configMap.get(Config.HTML_CHILD_URL_RULES);
1031 if (StringUtil.isBlank(ruleString)) {
1032 return childUrlRuleMap.entrySet().stream().map(e -> new Pair<>(e.getKey(), e.getValue()));
1033 }
1034 return Arrays.stream(ruleString.split(","))
1035 .map(s -> s.split(":"))
1036 .filter(v -> v.length == 2)
1037 .map(v -> new Pair<String, String>(v[0].trim(), v[1].trim()));
1038 }
1039
1040
1041
1042
1043
1044
1045
1046 @Override
1047 protected List<RequestData> convertChildUrlList(final List<RequestData> urlList) {
1048 if (urlList != null) {
1049 final PathMappingHelper pathMappingHelper = getPathMappingHelper();
1050 for (final RequestData requestData : urlList) {
1051 String url = requestData.getUrl();
1052 for (final Map.Entry<String, String> entry : convertUrlMap.entrySet()) {
1053 url = url.replaceAll(entry.getKey(), entry.getValue());
1054 }
1055 url = pathMappingHelper.replaceUrl(url);
1056 requestData.setUrl(replaceDuplicateHost(url));
1057 }
1058 }
1059 return urlList;
1060 }
1061
1062
1063
1064
1065
1066
1067 protected PathMappingHelper getPathMappingHelper() {
1068 return ComponentUtil.getPathMappingHelper();
1069 }
1070
1071
1072
1073
1074
1075
1076
1077 @Override
1078 public Object getData(final AccessResultData<?> accessResultData) {
1079 final byte[] data = accessResultData.getData();
1080 if (data != null) {
1081 try {
1082 return dataSerializer.fromBinaryToObject(data);
1083 } catch (final Exception e) {
1084 throw new CrawlerSystemException("Could not create an instanced from bytes.", e);
1085 }
1086 }
1087 return new HashMap<String, Object>();
1088 }
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098 @Override
1099 protected void addChildUrlFromTagAttribute(final List<String> urlList, final URL url, final String attrValue, final String encoding) {
1100 final String urlValue = attrValue.trim();
1101 String u = null;
1102 try {
1103 final URL childUrl = new URL(url, urlValue.startsWith(":") ? url.getProtocol() + urlValue : urlValue);
1104 String childUrlStr = childUrl.toExternalForm();
1105 final String path = childUrl.getPath();
1106 if (path != null && path.startsWith("/../")) {
1107 String normalizedPath = path;
1108 while (normalizedPath.startsWith("/../")) {
1109 normalizedPath = normalizedPath.substring(3);
1110 }
1111 if (!normalizedPath.startsWith("/")) {
1112 normalizedPath = "/" + normalizedPath;
1113 }
1114 childUrlStr = childUrl.getProtocol() + "://" + childUrl.getAuthority() + normalizedPath;
1115 if (childUrl.getQuery() != null) {
1116 childUrlStr += "?" + childUrl.getQuery();
1117 }
1118 }
1119 u = encodeUrl(normalizeUrl(childUrlStr), encoding);
1120 } catch (final MalformedURLException e) {
1121 final int pos = urlValue.indexOf(':');
1122 if (pos > 0 && pos < 10) {
1123 u = encodeUrl(normalizeUrl(urlValue), encoding);
1124 }
1125 }
1126
1127 if (u == null) {
1128 logger.warn("Ignored child URL: childUrl={}, parentUrl={}", attrValue, url);
1129 return;
1130 }
1131
1132 if (logger.isDebugEnabled()) {
1133 logger.debug("URL conversion: original={}, converted={}", attrValue, u);
1134 }
1135 if (StringUtil.isNotBlank(u)) {
1136 if (logger.isDebugEnabled()) {
1137 logger.debug("Adding child URL: url={}", u);
1138 }
1139 urlList.add(u);
1140 } else if (logger.isDebugEnabled()) {
1141 logger.debug("Skipping child URL: url={}", u);
1142 }
1143 }
1144
1145
1146
1147
1148
1149
1150
1151 private boolean isUtf8BomBytes(final byte[] b) {
1152 return b[0] == (byte) 0xEF && b[1] == (byte) 0xBB && b[2] == (byte) 0xBF;
1153 }
1154
1155
1156
1157
1158
1159
1160 public void setUseGoogleOffOn(final boolean useGoogleOffOn) {
1161 this.useGoogleOffOn = useGoogleOffOn;
1162 }
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172 protected String getThumbnailUrl(final ResponseData responseData, final Document document) {
1173
1174 try {
1175
1176 final Node thumbnailNode = getXPathAPI().selectSingleNode(document, META_NAME_THUMBNAIL_CONTENT);
1177 if (thumbnailNode != null) {
1178 final String content = thumbnailNode.getTextContent();
1179 if (StringUtil.isNotBlank(content)) {
1180 final URL thumbnailUrl = getURL(responseData.getUrl(), content);
1181 if (thumbnailUrl != null) {
1182 return thumbnailUrl.toExternalForm();
1183 }
1184 }
1185 }
1186
1187
1188 final Node ogImageNode = getXPathAPI().selectSingleNode(document, META_PROPERTY_OGIMAGE_CONTENT);
1189 if (ogImageNode != null) {
1190 final String content = ogImageNode.getTextContent();
1191 if (StringUtil.isNotBlank(content)) {
1192 final URL thumbnailUrl = getURL(responseData.getUrl(), content);
1193 if (thumbnailUrl != null) {
1194 return thumbnailUrl.toExternalForm();
1195 }
1196 }
1197 }
1198
1199 final XPathNodes imgNodeList = getXPathAPI().selectNodeList(document, fessConfig.getThumbnailHtmlImageXpath());
1200 String firstThumbnailUrl = null;
1201 for (int i = 0; i < imgNodeList.size(); i++) {
1202 final Node imgNode = imgNodeList.get(i);
1203 if (logger.isDebugEnabled()) {
1204 logger.debug("img tag: {}", imgNode);
1205 }
1206 final NamedNodeMap attributes = imgNode.getAttributes();
1207 final String thumbnailUrl = getThumbnailSrc(responseData.getUrl(), attributes);
1208 final Integer height = getAttributeAsInteger(attributes, "height");
1209 final Integer width = getAttributeAsInteger(attributes, "width");
1210 if (!fessConfig.isThumbnailHtmlImageUrl(thumbnailUrl)) {
1211 continue;
1212 }
1213 if (height != null && width != null) {
1214 try {
1215 if (fessConfig.validateThumbnailSize(width, height)) {
1216 return thumbnailUrl;
1217 }
1218 } catch (final Exception e) {
1219 logger.debug("Failed to parse {} at {}", imgNode, responseData.getUrl(), e);
1220 }
1221 } else if (firstThumbnailUrl == null) {
1222 firstThumbnailUrl = thumbnailUrl;
1223 }
1224 }
1225
1226 if (firstThumbnailUrl != null) {
1227 return firstThumbnailUrl;
1228 }
1229 } catch (final Exception e) {
1230 logger.warn("Failed to retrieve thumbnail url from {}", responseData.getUrl(), e);
1231 }
1232 return null;
1233 }
1234
1235
1236
1237
1238
1239
1240
1241
1242 protected String getThumbnailSrc(final String url, final NamedNodeMap attributes) {
1243 final Node srcNode = attributes.getNamedItem("src");
1244 if (srcNode != null) {
1245 try {
1246 final URL thumbnailUrl = getURL(url, srcNode.getTextContent());
1247 if (thumbnailUrl != null) {
1248 return thumbnailUrl.toExternalForm();
1249 }
1250 } catch (final Exception e) {
1251 if (logger.isDebugEnabled()) {
1252 logger.debug("Failed to parse thumbnail url for {} : {}", url, attributes, e);
1253 }
1254 }
1255 }
1256 return null;
1257 }
1258
1259
1260
1261
1262
1263
1264
1265
1266 protected Integer getAttributeAsInteger(final NamedNodeMap attributes, final String name) {
1267 final Node namedItem = attributes.getNamedItem(name);
1268 if (namedItem == null) {
1269 return null;
1270 }
1271 final String value = namedItem.getTextContent();
1272 if (value == null) {
1273 return null;
1274 }
1275 try {
1276 return Integer.parseInt(value);
1277 } catch (final NumberFormatException e) {
1278 if (value.endsWith("%") || value.endsWith("px")) {
1279 return null;
1280 }
1281 return 0;
1282 }
1283 }
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293 protected URL getURL(final String currentUrl, final String url) throws MalformedURLException {
1294 if (url != null) {
1295 if (url.startsWith("://")) {
1296 final String protocol = currentUrl.split(":")[0];
1297 return new URL(protocol + url);
1298 }
1299 if (url.startsWith("//")) {
1300 final String protocol = currentUrl.split(":")[0];
1301 return new URL(protocol + ":" + url);
1302 }
1303 if (url.startsWith("/") || url.indexOf(':') == -1) {
1304 return new URL(new URL(currentUrl), url);
1305 }
1306 return new URL(url);
1307 }
1308 return null;
1309 }
1310
1311
1312
1313
1314
1315
1316
1317
1318 public void addFieldRule(final String name, final String xpath, final boolean isPruned) {
1319 addFieldRule(name, xpath);
1320 fieldPrunedRuleMap.put(name, isPruned);
1321 }
1322
1323
1324
1325
1326
1327
1328 public void setConvertUrlMap(final Map<String, String> convertUrlMap) {
1329 this.convertUrlMap.putAll(convertUrlMap);
1330 }
1331
1332
1333
1334
1335
1336
1337
1338 public void addConvertUrl(final String regex, final String replacement) {
1339 convertUrlMap.put(regex, replacement);
1340 }
1341 }