1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedInputStream;
21 import java.net.MalformedURLException;
22 import java.net.URL;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.LinkedHashMap;
29 import java.util.List;
30 import java.util.Locale;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.stream.Collectors;
34
35 import javax.annotation.PostConstruct;
36 import javax.xml.transform.TransformerException;
37
38 import org.apache.logging.log4j.LogManager;
39 import org.apache.logging.log4j.Logger;
40 import org.apache.xpath.objects.XObject;
41 import org.codelibs.core.io.InputStreamUtil;
42 import org.codelibs.core.io.SerializeUtil;
43 import org.codelibs.core.lang.StringUtil;
44 import org.codelibs.core.misc.ValueHolder;
45 import org.codelibs.fess.Constants;
46 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
47 import org.codelibs.fess.crawler.entity.AccessResultData;
48 import org.codelibs.fess.crawler.entity.RequestData;
49 import org.codelibs.fess.crawler.entity.ResponseData;
50 import org.codelibs.fess.crawler.entity.ResultData;
51 import org.codelibs.fess.crawler.entity.UrlQueue;
52 import org.codelibs.fess.crawler.exception.ChildUrlsException;
53 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
54 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
55 import org.codelibs.fess.crawler.transformer.impl.XpathTransformer;
56 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
57 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
58 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
59 import org.codelibs.fess.es.config.exentity.CrawlingConfig.Param.Config;
60 import org.codelibs.fess.es.config.exentity.CrawlingConfig.Param.XPath;
61 import org.codelibs.fess.helper.CrawlingConfigHelper;
62 import org.codelibs.fess.helper.CrawlingInfoHelper;
63 import org.codelibs.fess.helper.DocumentHelper;
64 import org.codelibs.fess.helper.DuplicateHostHelper;
65 import org.codelibs.fess.helper.FileTypeHelper;
66 import org.codelibs.fess.helper.LabelTypeHelper;
67 import org.codelibs.fess.helper.PathMappingHelper;
68 import org.codelibs.fess.helper.SystemHelper;
69 import org.codelibs.fess.mylasta.direction.FessConfig;
70 import org.codelibs.fess.util.ComponentUtil;
71 import org.codelibs.fess.util.PrunedTag;
72 import org.codelibs.nekohtml.parsers.DOMParser;
73 import org.w3c.dom.Document;
74 import org.w3c.dom.NamedNodeMap;
75 import org.w3c.dom.Node;
76 import org.w3c.dom.NodeList;
77 import org.xml.sax.InputSource;
78
79 public class FessXpathTransformer extends XpathTransformer implements FessTransformer {
80
81 private static final Logger logger = LogManager.getLogger(FessXpathTransformer.class);
82
83 private static final String X_ROBOTS_TAG = "X-Robots-Tag";
84
85 private static final String META_NAME_THUMBNAIL_CONTENT = "//META[@name=\"thumbnail\" or @name=\"THUMBNAIL\"]/@content";
86
87 private static final String META_PROPERTY_OGIMAGE_CONTENT = "//META[@property=\"og:image\"]/@content";
88
89 private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";
90
91 private static final String ROBOTS_TAG_NONE = "none";
92
93 private static final String ROBOTS_TAG_NOINDEX = "noindex";
94
95 private static final String ROBOTS_TAG_NOFOLLOW = "nofollow";
96
97 private static final int UTF8_BOM_SIZE = 3;
98
99 public boolean prunedContent = true;
100
101 protected Map<String, String> convertUrlMap = new LinkedHashMap<>();
102
103 protected FessConfig fessConfig;
104
105 protected boolean useGoogleOffOn = true;
106
107 protected Map<String, Boolean> fieldPrunedRuleMap = new HashMap<>();
108
109 @PostConstruct
110 public void init() {
111 if (logger.isDebugEnabled()) {
112 logger.debug("Initialize {}", this.getClass().getSimpleName());
113 }
114 fessConfig = ComponentUtil.getFessConfig();
115 }
116
117 @Override
118 public FessConfig getFessConfig() {
119 return fessConfig;
120 }
121
122 @Override
123 public Logger getLogger() {
124 return logger;
125 }
126
127 @Override
128 protected void storeData(final ResponseData responseData, final ResultData resultData) {
129 final DOMParser parser = getDomParser();
130 try (final BufferedInputStream bis = new BufferedInputStream(responseData.getResponseBody())) {
131 final byte[] bomBytes = new byte[UTF8_BOM_SIZE];
132 bis.mark(UTF8_BOM_SIZE);
133 final int size = bis.read(bomBytes);
134 if (size < 3 || !isUtf8BomBytes(bomBytes)) {
135 bis.reset();
136 }
137 final InputSource is = new InputSource(bis);
138 if (responseData.getCharSet() != null) {
139 is.setEncoding(responseData.getCharSet());
140 }
141 parser.parse(is);
142 } catch (final Exception e) {
143 throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);
144 }
145
146 final Document document = parser.getDocument();
147
148 processMetaRobots(responseData, resultData, document);
149 processXRobotsTag(responseData, resultData);
150
151 final Map<String, Object> dataMap = new LinkedHashMap<>();
152 for (final Map.Entry<String, String> entry : fieldRuleMap.entrySet()) {
153 final String path = entry.getValue();
154 try {
155 final XObject xObj = getXPathAPI().eval(document, path);
156 final int type = xObj.getType();
157 switch (type) {
158 case XObject.CLASS_BOOLEAN:
159 final boolean b = xObj.bool();
160 putResultDataBody(dataMap, entry.getKey(), Boolean.toString(b));
161 break;
162 case XObject.CLASS_NUMBER:
163 final double d = xObj.num();
164 putResultDataBody(dataMap, entry.getKey(), Double.toString(d));
165 break;
166 case XObject.CLASS_STRING:
167 final String str = xObj.str();
168 putResultDataBody(dataMap, entry.getKey(), str);
169 break;
170 case XObject.CLASS_NULL:
171 case XObject.CLASS_UNKNOWN:
172 case XObject.CLASS_NODESET:
173 case XObject.CLASS_RTREEFRAG:
174 case XObject.CLASS_UNRESOLVEDVARIABLE:
175 default:
176 final Boolean isPruned = fieldPrunedRuleMap.get(entry.getKey());
177 Node value = getXPathAPI().selectSingleNode(document, entry.getValue());
178 if (value != null && isPruned != null && isPruned.booleanValue()) {
179 value = pruneNode(value);
180 }
181 putResultDataBody(dataMap, entry.getKey(), value != null ? value.getTextContent() : null);
182 break;
183 }
184 } catch (final TransformerException e) {
185 logger.warn("Could not parse a value of {}:{}", entry.getKey(), entry.getValue(), e);
186 }
187 }
188
189 putAdditionalData(dataMap, responseData, document);
190 normalizeData(responseData, dataMap);
191
192 try {
193 resultData.setData(SerializeUtil.fromObjectToBinary(dataMap));
194 } catch (final Exception e) {
195 throw new CrawlingAccessException("Could not serialize object: " + responseData.getUrl(), e);
196 }
197 resultData.setEncoding(charsetName);
198 }
199
200 protected void normalizeData(final ResponseData responseData, final Map<String, Object> dataMap) {
201 final Object titleObj = dataMap.get(fessConfig.getIndexFieldTitle());
202 if (titleObj != null) {
203 dataMap.put(fessConfig.getIndexFieldTitle(),
204 ComponentUtil.getDocumentHelper().getTitle(responseData, titleObj.toString(), dataMap));
205 }
206 }
207
208 protected void processMetaRobots(final ResponseData responseData, final ResultData resultData, final Document document) {
209 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
210 final String ignore = configMap.get(Config.IGNORE_ROBOTS_TAGS);
211 if (ignore == null) {
212 if (fessConfig.isCrawlerIgnoreRobotsTags()) {
213 return;
214 }
215 } else if (Boolean.parseBoolean(ignore)) {
216 return;
217 }
218
219
220 try {
221 final Node value = getXPathAPI().selectSingleNode(document, META_NAME_ROBOTS_CONTENT);
222 if (value != null) {
223 boolean noindex = false;
224 boolean nofollow = false;
225 final String content = value.getTextContent().toLowerCase(Locale.ROOT);
226 if (content.contains(ROBOTS_TAG_NONE)) {
227 noindex = true;
228 nofollow = true;
229 } else {
230 if (content.contains(ROBOTS_TAG_NOINDEX)) {
231 noindex = true;
232 }
233 if (content.contains(ROBOTS_TAG_NOFOLLOW)) {
234 nofollow = true;
235 }
236 }
237 if (noindex && nofollow) {
238 logger.info("META(robots=noindex,nofollow): {}", responseData.getUrl());
239 throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots");
240 }
241 if (noindex) {
242 logger.info("META(robots=noindex): {}", responseData.getUrl());
243 storeChildUrls(responseData, resultData);
244 throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots");
245 }
246 if (nofollow) {
247 logger.info("META(robots=nofollow): {}", responseData.getUrl());
248 responseData.setNoFollow(true);
249 }
250 }
251 } catch (final TransformerException e) {
252 logger.warn("Could not parse a value of {}", META_NAME_ROBOTS_CONTENT, e);
253 }
254
255 }
256
257 protected void processXRobotsTag(final ResponseData responseData, final ResultData resultData) {
258 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
259 final String ignore = configMap.get(Config.IGNORE_ROBOTS_TAGS);
260 if (ignore == null) {
261 if (fessConfig.isCrawlerIgnoreRobotsTags()) {
262 return;
263 }
264 } else if (Boolean.parseBoolean(ignore)) {
265 return;
266 }
267
268
269 responseData.getMetaDataMap().entrySet().stream().filter(e -> X_ROBOTS_TAG.equalsIgnoreCase(e.getKey()) && e.getValue() != null)
270 .forEach(e -> {
271 boolean noindex = false;
272 boolean nofollow = false;
273 final String value = e.getValue().toString().toLowerCase(Locale.ROOT);
274 if (value.contains(ROBOTS_TAG_NONE)) {
275 noindex = true;
276 nofollow = true;
277 } else {
278 if (value.contains(ROBOTS_TAG_NOINDEX)) {
279 noindex = true;
280 }
281 if (value.contains(ROBOTS_TAG_NOFOLLOW)) {
282 nofollow = true;
283 }
284 }
285 if (noindex && nofollow) {
286 logger.info("HEADER(robots=noindex,nofollow): {}", responseData.getUrl());
287 throw new ChildUrlsException(Collections.emptySet(), "#processXRobotsTag");
288 }
289 if (noindex) {
290 logger.info("HEADER(robots=noindex): {}", responseData.getUrl());
291 storeChildUrls(responseData, resultData);
292 throw new ChildUrlsException(resultData.getChildUrlSet(), "#processXRobotsTag");
293 }
294 if (nofollow) {
295 logger.info("HEADER(robots=nofollow): {}", responseData.getUrl());
296 responseData.setNoFollow(true);
297 }
298 });
299 }
300
301 protected Map<String, String> getConfigPrameterMap(final ResponseData responseData, final ConfigName config) {
302 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
303 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
304 final Map<String, String> configMap = crawlingConfig.getConfigParameterMap(config);
305 return configMap;
306 }
307
308 protected boolean isValidUrl(final String urlStr) {
309 if (StringUtil.isBlank(urlStr)) {
310 return false;
311 }
312 final String value;
313 if (urlStr.startsWith("://")) {
314 value = "http" + urlStr;
315 } else if (urlStr.startsWith("//")) {
316 value = "http:" + urlStr;
317 } else {
318 value = urlStr;
319 }
320 try {
321 final URL url = new java.net.URL(value);
322 final String host = url.getHost();
323 if (StringUtil.isBlank(host)) {
324 return false;
325 }
326 if ("http".equalsIgnoreCase(host) || "https".equalsIgnoreCase(host)) {
327 return false;
328 }
329 } catch (final MalformedURLException e) {
330 return false;
331 }
332 return true;
333 }
334
335 protected boolean isValidCanonicalUrl(final String url, final String canonicalUrl) {
336 if (url.startsWith("https:") && canonicalUrl.startsWith("http:")) {
337 if (logger.isDebugEnabled()) {
338 logger.debug("Invalid Canonical Url(https->http): {} -> {}", url, canonicalUrl);
339 }
340 return false;
341 }
342 return true;
343 }
344
345 protected void putAdditionalData(final Map<String, Object> dataMap, final ResponseData responseData, final Document document) {
346
347 final String canonicalUrl = getCanonicalUrl(responseData, document);
348 if (canonicalUrl != null && !canonicalUrl.equals(responseData.getUrl()) && isValidUrl(canonicalUrl)
349 && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {
350 final Set<RequestData> childUrlSet = new HashSet<>();
351 childUrlSet.add(RequestDataBuilder.newRequestData().get().url(canonicalUrl).build());
352 logger.info("CANONICAL: {} -> {}", responseData.getUrl(), canonicalUrl);
353 throw new ChildUrlsException(childUrlSet, this.getClass().getName() + "#putAdditionalData");
354 }
355
356 final FessConfig fessConfig = ComponentUtil.getFessConfig();
357 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
358 final String sessionId = crawlingInfoHelper.getCanonicalSessionId(responseData.getSessionId());
359 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
360 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
361 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
362 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
363 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
364 final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
365 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
366 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
367 String url = responseData.getUrl();
368 final String indexingTarget = crawlingConfig.getIndexingTarget(url);
369 url = pathMappingHelper.replaceUrl(sessionId, url);
370 final String mimeType = responseData.getMimeType();
371
372 final Map<String, String> fieldConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.FIELD);
373 final Map<String, String> xpathConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.XPATH);
374
375 String urlEncoding;
376 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
377 if (urlQueue != null && urlQueue.getEncoding() != null) {
378 urlEncoding = urlQueue.getEncoding();
379 } else {
380 urlEncoding = responseData.getCharSet();
381 }
382
383
384 final String configId = crawlingConfig.getConfigId();
385 if (configId != null) {
386 putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
387 }
388
389 if (documentExpires != null) {
390 putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), documentExpires);
391 }
392
393 final String lang = systemHelper.normalizeHtmlLang(getSingleNodeValue(document, getLangXpath(fessConfig, xpathConfigMap), true));
394 if (lang != null) {
395 putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), lang);
396 }
397
398
399 final String body = getSingleNodeValue(document, getContentXpath(fessConfig, xpathConfigMap), prunedContent);
400 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(),
401 documentHelper.getContent(crawlingConfig, responseData, body, dataMap));
402 if ((Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache()))
403 || fessConfig.isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
404 if (responseData.getContentLength() > 0
405 && responseData.getContentLength() <= fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger().longValue()) {
406 String charSet = responseData.getCharSet();
407 if (charSet == null) {
408 charSet = Constants.UTF_8;
409 }
410 try (final BufferedInputStream is = new BufferedInputStream(responseData.getResponseBody())) {
411
412 putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), new String(InputStreamUtil.getBytes(is), charSet));
413 putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
414 } catch (final Exception e) {
415 logger.warn("Failed to write a cache: {}:{}", sessionId, responseData, e);
416 }
417 } else {
418 logger.debug("Content size is too large({} > {}): {}", responseData.getContentLength(),
419 fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger(), responseData.getUrl());
420 }
421 }
422
423 final String digest = getSingleNodeValue(document, getDigestXpath(fessConfig, xpathConfigMap), false);
424 if (StringUtil.isNotBlank(digest)) {
425 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(), digest);
426 } else {
427 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
428 documentHelper.getDigest(responseData, body, dataMap, fessConfig.getCrawlerDocumentHtmlMaxDigestLengthAsInteger()));
429 }
430
431 putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
432
433 putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHost(url));
434
435 putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSite(url, urlEncoding));
436
437 final String fileName = getFileName(url, urlEncoding);
438 if (StringUtil.isNotBlank(fileName)) {
439 putResultDataBody(dataMap, fessConfig.getIndexFieldFilename(), fileName);
440 }
441
442 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
443
444 final Date now = systemHelper.getCurrentTime();
445 putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), now);
446
447 putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), getAnchorList(document, responseData));
448
449 putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
450 if (fileTypeHelper != null) {
451
452 putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
453 }
454
455 putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
456
457 final Date lastModified = responseData.getLastModified();
458 if (lastModified != null) {
459 putResultDataBody(dataMap, fessConfig.getIndexFieldLastModified(), lastModified);
460
461 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), lastModified);
462 } else {
463
464 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), now);
465 }
466
467 putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
468
469 putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
470
471 putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeHelper.getMatchedLabelValueSet(url));
472
473 final List<String> roleTypeList = new ArrayList<>();
474 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
475 putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
476
477 putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
478 stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).collect(Collectors.toList())));
479
480 putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
481
482 String parentUrl = responseData.getParentUrl();
483 if (StringUtil.isNotBlank(parentUrl)) {
484 parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
485 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
486 putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
487 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
488 }
489
490 final String thumbnailUrl = getThumbnailUrl(responseData, document);
491 if (StringUtil.isNotBlank(thumbnailUrl)) {
492 putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), thumbnailUrl);
493 }
494
495
496 final String scriptType = crawlingConfig.getScriptType();
497 final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);
498 xpathConfigMap.entrySet().stream().filter(e -> !e.getKey().startsWith("default.")).forEach(e -> {
499 final String key = e.getKey();
500 final String value = getSingleNodeValue(document, e.getValue(), true);
501 putResultDataWithTemplate(dataMap, key, value, scriptConfigMap.get(key), scriptType);
502 });
503 crawlingConfig.getConfigParameterMap(ConfigName.VALUE).entrySet().stream().forEach(e -> {
504 final String key = e.getKey();
505 final String value = e.getValue();
506 putResultDataWithTemplate(dataMap, key, value, scriptConfigMap.get(key), scriptType);
507 });
508 }
509
510 protected String getLangXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
511 final String xpath = xpathConfigMap.get(XPath.DEFAULT_LANG);
512 if (StringUtil.isNotBlank(xpath)) {
513 return xpath;
514 }
515 return fessConfig.getCrawlerDocumentHtmlLangXpath();
516 }
517
518 protected String getContentXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
519 final String xpath = xpathConfigMap.get(XPath.DEFAULT_CONTENT);
520 if (StringUtil.isNotBlank(xpath)) {
521 return xpath;
522 }
523 return fessConfig.getCrawlerDocumentHtmlContentXpath();
524 }
525
526 protected String getDigestXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
527 final String xpath = xpathConfigMap.get(XPath.DEFAULT_DIGEST);
528 if (StringUtil.isNotBlank(xpath)) {
529 return xpath;
530 }
531 return fessConfig.getCrawlerDocumentHtmlDigestXpath();
532 }
533
534 protected String getCanonicalUrl(final ResponseData responseData, final Document document) {
535 final Map<String, String> configMap = getConfigPrameterMap(responseData, ConfigName.CONFIG);
536 String xpath = configMap.get(Config.HTML_CANONICAL_XPATH);
537 if (xpath == null) {
538 xpath = fessConfig.getCrawlerDocumentHtmlCanonicalXpath();
539 }
540 if (StringUtil.isBlank(xpath)) {
541 return null;
542 }
543 final String canonicalUrl = getSingleNodeValue(document, xpath, false);
544 if (StringUtil.isBlank(canonicalUrl)) {
545 return null;
546 }
547 return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
548 }
549
550 protected String normalizeCanonicalUrl(final String baseUrl, final String canonicalUrl) {
551 try {
552 final URL u = new URL(baseUrl);
553 return new URL(u, canonicalUrl.startsWith(":") ? u.getProtocol() + canonicalUrl : canonicalUrl).toString();
554 } catch (final MalformedURLException e) {
555 logger.warn("Invalid canonical url: {} : {}", baseUrl, canonicalUrl, e);
556 }
557 return null;
558 }
559
560 protected String removeCommentTag(final String content) {
561 if (content == null) {
562 return StringUtil.EMPTY;
563 }
564 String value = content;
565 int pos = value.indexOf("<!--");
566 while (pos >= 0) {
567 final int lastPos = value.indexOf("-->", pos);
568 if (lastPos < 0) {
569 break;
570 }
571 if (pos == 0) {
572 value = " " + value.substring(lastPos + 3);
573 } else {
574 value = value.substring(0, pos) + " " + value.substring(lastPos + 3);
575 }
576 pos = value.indexOf("<!--");
577 }
578 return value;
579 }
580
581 protected String getSingleNodeValue(final Document document, final String xpath, final boolean pruned) {
582 StringBuilder buf = null;
583 NodeList list = null;
584 try {
585 list = getXPathAPI().selectNodeList(document, xpath);
586 for (int i = 0; i < list.getLength(); i++) {
587 if (buf == null) {
588 buf = new StringBuilder(1000);
589 }
590 Node node = list.item(i).cloneNode(true);
591 if (useGoogleOffOn) {
592 node = processGoogleOffOn(node, new ValueHolder<>(true));
593 }
594 if (pruned) {
595 node = pruneNode(node);
596 }
597 parseTextContent(node, buf);
598 }
599 } catch (final Exception e) {
600 logger.warn("Could not parse a value of {}", xpath);
601 }
602 if (buf == null) {
603 return null;
604 }
605 return buf.toString().trim();
606 }
607
608 protected void parseTextContent(final Node node, final StringBuilder buf) {
609 if (node.hasChildNodes()) {
610 final NodeList nodeList = node.getChildNodes();
611 for (int i = 0; i < nodeList.getLength(); i++) {
612 final Node childNode = nodeList.item(i);
613 parseTextContent(childNode, buf);
614 }
615 } else if (node.getNodeType() == Node.TEXT_NODE) {
616 final String value = node.getTextContent();
617 if (value != null) {
618 final String content = value.trim();
619 if (content.length() > 0) {
620 buf.append(' ').append(content);
621 }
622 }
623 }
624 }
625
626 protected Node processGoogleOffOn(final Node node, final ValueHolder<Boolean> flag) {
627 final NodeList nodeList = node.getChildNodes();
628 List<Node> removedNodeList = null;
629 for (int i = 0; i < nodeList.getLength(); i++) {
630 final Node childNode = nodeList.item(i);
631 if (childNode.getNodeType() == Node.COMMENT_NODE) {
632 final String comment = childNode.getNodeValue().trim();
633 if (comment.startsWith("googleoff:")) {
634 flag.setValue(false);
635 } else if (comment.startsWith("googleon:")) {
636 flag.setValue(true);
637 }
638 }
639
640 if (!flag.getValue() && childNode.getNodeType() == Node.TEXT_NODE) {
641 if (removedNodeList == null) {
642 removedNodeList = new ArrayList<>();
643 }
644 removedNodeList.add(childNode);
645 } else {
646 processGoogleOffOn(childNode, flag);
647 }
648 }
649
650 if (removedNodeList != null) {
651 removedNodeList.stream().forEach(n -> node.removeChild(n));
652 }
653
654 return node;
655 }
656
657 protected Node pruneNode(final Node node) {
658 final NodeList nodeList = node.getChildNodes();
659 final List<Node> childNodeList = new ArrayList<>();
660 final List<Node> removedNodeList = new ArrayList<>();
661 for (int i = 0; i < nodeList.getLength(); i++) {
662 final Node childNode = nodeList.item(i);
663 if (isPrunedTag(childNode)) {
664 removedNodeList.add(childNode);
665 } else {
666 childNodeList.add(childNode);
667 }
668 }
669
670 for (final Node childNode : removedNodeList) {
671 node.removeChild(childNode);
672 }
673
674 for (final Node childNode : childNodeList) {
675 pruneNode(childNode);
676 }
677
678 return node;
679 }
680
681 protected boolean isPrunedTag(final Node node) {
682 for (final PrunedTag prunedTag : fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray()) {
683 if (prunedTag.matches(node)) {
684 return true;
685 }
686 }
687 return false;
688 }
689
690 protected String getMultipleNodeValue(final Document document, final String xpath) {
691 NodeList nodeList = null;
692 final StringBuilder buf = new StringBuilder(100);
693 try {
694 nodeList = getXPathAPI().selectNodeList(document, xpath);
695 for (int i = 0; i < nodeList.getLength(); i++) {
696 final Node node = nodeList.item(i);
697 buf.append(node.getTextContent());
698 buf.append("\n");
699 }
700 } catch (final Exception e) {
701 logger.warn("Could not parse a value of {}", xpath, e);
702 }
703 return buf.toString().trim();
704 }
705
706 protected String replaceDuplicateHost(final String url) {
707 try {
708
709 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
710 return duplicateHostHelper.convert(url);
711 } catch (final Exception e) {
712 return url;
713 }
714 }
715
716 protected List<String> getAnchorList(final Document document, final ResponseData responseData) {
717 List<RequestData> anchorList = new ArrayList<>();
718 final String baseHref = getBaseHref(document);
719 try {
720 final URL url = getBaseUrl(responseData.getUrl(), baseHref);
721 for (final Map.Entry<String, String> entry : childUrlRuleMap.entrySet()) {
722 for (final String u : getUrlFromTagAttribute(url, document, entry.getKey(), entry.getValue(), responseData.getCharSet())) {
723 anchorList.add(RequestDataBuilder.newRequestData().get().url(u).build());
724 }
725 }
726 anchorList = convertChildUrlList(anchorList);
727 } catch (final Exception e) {
728 logger.warn("Could not parse anchor tags.", e);
729 }
730
731 final List<String> urlList = new ArrayList<>(anchorList.size());
732 for (final RequestData requestData : anchorList) {
733 urlList.add(requestData.getUrl());
734 }
735 return urlList;
736 }
737
738 protected URL getBaseUrl(final String currentUrl, final String baseHref) throws MalformedURLException {
739 if (baseHref != null) {
740 return getURL(currentUrl, baseHref);
741 }
742 return new URL(currentUrl);
743 }
744
745 @Override
746 protected List<RequestData> convertChildUrlList(final List<RequestData> urlList) {
747 if (urlList != null) {
748 final PathMappingHelper pathMappingHelper = getPathMappingHelper();
749 for (final RequestData requestData : urlList) {
750 String url = requestData.getUrl();
751 for (final Map.Entry<String, String> entry : convertUrlMap.entrySet()) {
752 url = url.replaceAll(entry.getKey(), entry.getValue());
753 }
754 url = pathMappingHelper.replaceUrl(url);
755 requestData.setUrl(replaceDuplicateHost(url));
756 }
757 }
758 return urlList;
759 }
760
761 protected PathMappingHelper getPathMappingHelper() {
762 return ComponentUtil.getPathMappingHelper();
763 }
764
765 @Override
766 public Object getData(final AccessResultData<?> accessResultData) {
767 final byte[] data = accessResultData.getData();
768 if (data != null) {
769 try {
770 return SerializeUtil.fromBinaryToObject(data);
771 } catch (final Exception e) {
772 throw new CrawlerSystemException("Could not create an instanced from bytes.", e);
773 }
774 }
775 return new HashMap<String, Object>();
776 }
777
778 @Override
779 protected void addChildUrlFromTagAttribute(final List<String> urlList, final URL url, final String attrValue, final String encoding) {
780 final String urlValue = attrValue.trim();
781 URL childUrl;
782 String u = null;
783 try {
784 childUrl = new URL(url, urlValue.startsWith(":") ? url.getProtocol() + urlValue : urlValue);
785 u = encodeUrl(normalizeUrl(childUrl.toExternalForm()), encoding);
786 } catch (final MalformedURLException e) {
787 final int pos = urlValue.indexOf(':');
788 if (pos > 0 && pos < 10) {
789 u = encodeUrl(normalizeUrl(urlValue), encoding);
790 }
791 }
792
793 if (u == null) {
794 logger.warn("Ignored child URL: {} in {}", attrValue, url);
795 return;
796 }
797
798 if (logger.isDebugEnabled()) {
799 logger.debug("{} -> {}", attrValue, u);
800 }
801 if (StringUtil.isNotBlank(u)) {
802 if (logger.isDebugEnabled()) {
803 logger.debug("Add Child: {}", u);
804 }
805 urlList.add(u);
806 } else if (logger.isDebugEnabled()) {
807 logger.debug("Skip Child: {}", u);
808 }
809 }
810
811 private boolean isUtf8BomBytes(final byte[] b) {
812 return b[0] == (byte) 0xEF && b[1] == (byte) 0xBB && b[2] == (byte) 0xBF;
813 }
814
815 public void setUseGoogleOffOn(final boolean useGoogleOffOn) {
816 this.useGoogleOffOn = useGoogleOffOn;
817 }
818
819 protected String getThumbnailUrl(final ResponseData responseData, final Document document) {
820
821 try {
822
823 final Node thumbnailNode = getXPathAPI().selectSingleNode(document, META_NAME_THUMBNAIL_CONTENT);
824 if (thumbnailNode != null) {
825 final String content = thumbnailNode.getTextContent();
826 if (StringUtil.isNotBlank(content)) {
827 final URL thumbnailUrl = getURL(responseData.getUrl(), content);
828 if (thumbnailUrl != null) {
829 return thumbnailUrl.toExternalForm();
830 }
831 }
832 }
833
834
835 final Node ogImageNode = getXPathAPI().selectSingleNode(document, META_PROPERTY_OGIMAGE_CONTENT);
836 if (ogImageNode != null) {
837 final String content = ogImageNode.getTextContent();
838 if (StringUtil.isNotBlank(content)) {
839 final URL thumbnailUrl = getURL(responseData.getUrl(), content);
840 if (thumbnailUrl != null) {
841 return thumbnailUrl.toExternalForm();
842 }
843 }
844 }
845
846 final NodeList imgNodeList = getXPathAPI().selectNodeList(document, fessConfig.getThumbnailHtmlImageXpath());
847 String firstThumbnailUrl = null;
848 for (int i = 0; i < imgNodeList.getLength(); i++) {
849 final Node imgNode = imgNodeList.item(i);
850 if (logger.isDebugEnabled()) {
851 logger.debug("img tag: {}", imgNode);
852 }
853 final NamedNodeMap attributes = imgNode.getAttributes();
854 final String thumbnailUrl = getThumbnailSrc(responseData.getUrl(), attributes);
855 final Integer height = getAttributeAsInteger(attributes, "height");
856 final Integer width = getAttributeAsInteger(attributes, "width");
857 if (!fessConfig.isThumbnailHtmlImageUrl(thumbnailUrl)) {
858 continue;
859 }
860 if (height != null && width != null) {
861 try {
862 if (fessConfig.validateThumbnailSize(width, height)) {
863 return thumbnailUrl;
864 }
865 } catch (final Exception e) {
866 logger.debug("Failed to parse {} at {}", imgNode, responseData.getUrl(), e);
867 }
868 } else if (firstThumbnailUrl == null) {
869 firstThumbnailUrl = thumbnailUrl;
870 }
871 }
872
873 if (firstThumbnailUrl != null) {
874 return firstThumbnailUrl;
875 }
876 } catch (final Exception e) {
877 logger.warn("Failed to retrieve thumbnail url from {}", responseData.getUrl(), e);
878 }
879 return null;
880 }
881
882 protected String getThumbnailSrc(final String url, final NamedNodeMap attributes) {
883 final Node srcNode = attributes.getNamedItem("src");
884 if (srcNode != null) {
885 try {
886 final URL thumbnailUrl = getURL(url, srcNode.getTextContent());
887 if (thumbnailUrl != null) {
888 return thumbnailUrl.toExternalForm();
889 }
890 } catch (final Exception e) {
891 if (logger.isDebugEnabled()) {
892 logger.debug("Failed to parse thumbnail url for {} : {}", url, attributes, e);
893 }
894 }
895 }
896 return null;
897 }
898
899 protected Integer getAttributeAsInteger(final NamedNodeMap attributes, final String name) {
900 final Node namedItem = attributes.getNamedItem(name);
901 if (namedItem == null) {
902 return null;
903 }
904 final String value = namedItem.getTextContent();
905 if (value == null) {
906 return null;
907 }
908 try {
909 return Integer.parseInt(value);
910 } catch (final NumberFormatException e) {
911 if (value.endsWith("%") || value.endsWith("px")) {
912 return null;
913 }
914 return 0;
915 }
916 }
917
918 protected URL getURL(final String currentUrl, final String url) throws MalformedURLException {
919 if (url != null) {
920 if (url.startsWith("://")) {
921 final String protocol = currentUrl.split(":")[0];
922 return new URL(protocol + url);
923 }
924 if (url.startsWith("//")) {
925 final String protocol = currentUrl.split(":")[0];
926 return new URL(protocol + ":" + url);
927 }
928 if (url.startsWith("/") || url.indexOf(':') == -1) {
929 return new URL(new URL(currentUrl), url);
930 }
931 return new URL(url);
932 }
933 return null;
934 }
935
936 public void addFieldRule(final String name, final String xpath, final boolean isPruned) {
937 addFieldRule(name, xpath);
938 fieldPrunedRuleMap.put(name, isPruned);
939 }
940
941 public void setConvertUrlMap(final Map<String, String> convertUrlMap) {
942 this.convertUrlMap.putAll(convertUrlMap);
943 }
944
945 public void addConvertUrl(final String regex, final String replacement) {
946 this.convertUrlMap.put(regex, replacement);
947 }
948 }