1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.crawler.transformer;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.BufferedInputStream;
21 import java.net.MalformedURLException;
22 import java.net.URL;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.HashMap;
27 import java.util.HashSet;
28 import java.util.LinkedHashMap;
29 import java.util.List;
30 import java.util.Locale;
31 import java.util.Map;
32 import java.util.Set;
33
34 import javax.annotation.PostConstruct;
35 import javax.xml.transform.TransformerException;
36
37 import org.apache.xpath.objects.XObject;
38 import org.codelibs.core.io.InputStreamUtil;
39 import org.codelibs.core.io.SerializeUtil;
40 import org.codelibs.core.lang.StringUtil;
41 import org.codelibs.core.misc.ValueHolder;
42 import org.codelibs.fess.Constants;
43 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
44 import org.codelibs.fess.crawler.entity.AccessResultData;
45 import org.codelibs.fess.crawler.entity.RequestData;
46 import org.codelibs.fess.crawler.entity.ResponseData;
47 import org.codelibs.fess.crawler.entity.ResultData;
48 import org.codelibs.fess.crawler.entity.UrlQueue;
49 import org.codelibs.fess.crawler.exception.ChildUrlsException;
50 import org.codelibs.fess.crawler.exception.CrawlerSystemException;
51 import org.codelibs.fess.crawler.exception.CrawlingAccessException;
52 import org.codelibs.fess.crawler.transformer.impl.XpathTransformer;
53 import org.codelibs.fess.crawler.util.CrawlingParameterUtil;
54 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
55 import org.codelibs.fess.es.config.exentity.CrawlingConfig.ConfigName;
56 import org.codelibs.fess.helper.CrawlingConfigHelper;
57 import org.codelibs.fess.helper.CrawlingInfoHelper;
58 import org.codelibs.fess.helper.DocumentHelper;
59 import org.codelibs.fess.helper.DuplicateHostHelper;
60 import org.codelibs.fess.helper.FileTypeHelper;
61 import org.codelibs.fess.helper.LabelTypeHelper;
62 import org.codelibs.fess.helper.PathMappingHelper;
63 import org.codelibs.fess.helper.SystemHelper;
64 import org.codelibs.fess.mylasta.direction.FessConfig;
65 import org.codelibs.fess.util.ComponentUtil;
66 import org.codelibs.fess.util.PrunedTag;
67 import org.cyberneko.html.parsers.DOMParser;
68 import org.slf4j.Logger;
69 import org.slf4j.LoggerFactory;
70 import org.w3c.dom.Document;
71 import org.w3c.dom.NamedNodeMap;
72 import org.w3c.dom.Node;
73 import org.w3c.dom.NodeList;
74 import org.xml.sax.InputSource;
75
76 public class FessXpathTransformer extends XpathTransformer implements FessTransformer {
77 private static final Logger logger = LoggerFactory.getLogger(FessXpathTransformer.class);
78
79 private static final String META_NAME_THUMBNAIL_CONTENT = "//META[@name=\"thumbnail\" or @name=\"THUMBNAIL\"]/@content";
80
81 private static final String META_PROPERTY_OGIMAGE_CONTENT = "//META[@property=\"og:image\"]/@content";
82
83 private static final String META_NAME_ROBOTS_CONTENT = "//META[@name=\"robots\" or @name=\"ROBOTS\"]/@content";
84
85 private static final String META_ROBOTS_NONE = "none";
86
87 private static final String META_ROBOTS_NOINDEX = "noindex";
88
89 private static final String META_ROBOTS_NOFOLLOW = "nofollow";
90
91 private static final int UTF8_BOM_SIZE = 3;
92
93 public boolean prunedContent = true;
94
95 public Map<String, String> convertUrlMap = new HashMap<>();
96
97 protected FessConfig fessConfig;
98
99 protected boolean useGoogleOffOn = true;
100
101 @PostConstruct
102 public void init() {
103 fessConfig = ComponentUtil.getFessConfig();
104 }
105
106 @Override
107 public FessConfig getFessConfig() {
108 return fessConfig;
109 }
110
111 @Override
112 public Logger getLogger() {
113 return logger;
114 }
115
116 @Override
117 protected void storeData(final ResponseData responseData, final ResultData resultData) {
118 final DOMParser parser = getDomParser();
119 try (final BufferedInputStream bis = new BufferedInputStream(responseData.getResponseBody())) {
120 final byte[] bomBytes = new byte[UTF8_BOM_SIZE];
121 bis.mark(UTF8_BOM_SIZE);
122 final int size = bis.read(bomBytes);
123 if (size < 3 || !isUtf8BomBytes(bomBytes)) {
124 bis.reset();
125 }
126 final InputSource is = new InputSource(bis);
127 if (responseData.getCharSet() != null) {
128 is.setEncoding(responseData.getCharSet());
129 }
130 parser.parse(is);
131 } catch (final Exception e) {
132 throw new CrawlingAccessException("Could not parse " + responseData.getUrl(), e);
133 }
134
135 final Document document = parser.getDocument();
136
137 if (!fessConfig.isCrawlerIgnoreMetaRobots()) {
138 processMetaRobots(responseData, resultData, document);
139 }
140
141 final Map<String, Object> dataMap = new LinkedHashMap<>();
142 for (final Map.Entry<String, String> entry : fieldRuleMap.entrySet()) {
143 final String path = entry.getValue();
144 try {
145 final XObject xObj = getXPathAPI().eval(document, path);
146 final int type = xObj.getType();
147 switch (type) {
148 case XObject.CLASS_BOOLEAN:
149 final boolean b = xObj.bool();
150 putResultDataBody(dataMap, entry.getKey(), Boolean.toString(b));
151 break;
152 case XObject.CLASS_NUMBER:
153 final double d = xObj.num();
154 putResultDataBody(dataMap, entry.getKey(), Double.toString(d));
155 break;
156 case XObject.CLASS_STRING:
157 final String str = xObj.str();
158 putResultDataBody(dataMap, entry.getKey(), str);
159 break;
160 case XObject.CLASS_NULL:
161 case XObject.CLASS_UNKNOWN:
162 case XObject.CLASS_NODESET:
163 case XObject.CLASS_RTREEFRAG:
164 case XObject.CLASS_UNRESOLVEDVARIABLE:
165 default:
166 final Node value = getXPathAPI().selectSingleNode(document, entry.getValue());
167 putResultDataBody(dataMap, entry.getKey(), value != null ? value.getTextContent() : null);
168 break;
169 }
170 } catch (final TransformerException e) {
171 logger.warn("Could not parse a value of " + entry.getKey() + ":" + entry.getValue(), e);
172 }
173 }
174
175 putAdditionalData(dataMap, responseData, document);
176
177 try {
178 resultData.setData(SerializeUtil.fromObjectToBinary(dataMap));
179 } catch (final Exception e) {
180 throw new CrawlingAccessException("Could not serialize object: " + responseData.getUrl(), e);
181 }
182 resultData.setEncoding(charsetName);
183 }
184
185 protected void processMetaRobots(final ResponseData responseData, final ResultData resultData, final Document document) {
186 try {
187 final Node value = getXPathAPI().selectSingleNode(document, META_NAME_ROBOTS_CONTENT);
188 if (value != null) {
189 final String content = value.getTextContent().toLowerCase(Locale.ROOT);
190 boolean noindex = false;
191 boolean nofollow = false;
192 if (content.contains(META_ROBOTS_NONE)) {
193 noindex = true;
194 nofollow = true;
195 } else {
196 if (content.contains(META_ROBOTS_NOINDEX)) {
197 noindex = true;
198 }
199 if (content.contains(META_ROBOTS_NOFOLLOW)) {
200 nofollow = true;
201 }
202 }
203
204 if (noindex && nofollow) {
205 logger.info("META(robots=noindex,nofollow): " + responseData.getUrl());
206 throw new ChildUrlsException(Collections.emptySet(), "#processMetaRobots(Document)");
207 } else if (noindex) {
208 logger.info("META(robots=noindex): " + responseData.getUrl());
209 storeChildUrls(responseData, resultData);
210 throw new ChildUrlsException(resultData.getChildUrlSet(), "#processMetaRobots(Document)");
211 } else if (nofollow) {
212 logger.info("META(robots=nofollow): " + responseData.getUrl());
213 responseData.setNoFollow(true);
214 }
215 }
216 } catch (final TransformerException e) {
217 logger.warn("Could not parse a value of " + META_NAME_ROBOTS_CONTENT, e);
218 }
219
220 }
221
222 protected boolean isValidUrl(final String urlStr) {
223 if (StringUtil.isBlank(urlStr)) {
224 return false;
225 }
226 final String value;
227 if (urlStr.startsWith("://")) {
228 value = "http" + urlStr;
229 } else if (urlStr.startsWith("//")) {
230 value = "http:" + urlStr;
231 } else {
232 value = urlStr;
233 }
234 try {
235 final URL url = new java.net.URL(value);
236 final String host = url.getHost();
237 if (StringUtil.isBlank(host)) {
238 return false;
239 }
240 if ("http".equalsIgnoreCase(host) || "https".equalsIgnoreCase(host)) {
241 return false;
242 }
243 } catch (final MalformedURLException e) {
244 return false;
245 }
246 return true;
247 }
248
249 protected boolean isValidCanonicalUrl(final String url, final String canonicalUrl) {
250 if (url.startsWith("https:") && canonicalUrl.startsWith("http:")) {
251 if (logger.isDebugEnabled()) {
252 logger.debug("Invalid Canonical Url(https->http): " + url + " -> " + canonicalUrl);
253 }
254 return false;
255 }
256 return true;
257 }
258
259 protected void putAdditionalData(final Map<String, Object> dataMap, final ResponseData responseData, final Document document) {
260
261 if (StringUtil.isNotBlank(fessConfig.getCrawlerDocumentHtmlCanonicalXpath())) {
262 final String canonicalUrl = getCanonicalUrl(responseData, document);
263 if (canonicalUrl != null && !canonicalUrl.equals(responseData.getUrl()) && isValidUrl(canonicalUrl)
264 && isValidCanonicalUrl(responseData.getUrl(), canonicalUrl)) {
265 final Set<RequestData> childUrlSet = new HashSet<>();
266 childUrlSet.add(RequestDataBuilder.newRequestData().get().url(canonicalUrl).build());
267 logger.info("CANONICAL: " + responseData.getUrl() + " -> " + canonicalUrl);
268 throw new ChildUrlsException(childUrlSet, this.getClass().getName()
269 + "#putAdditionalData(Map<String, Object>, ResponseData, Document)");
270 }
271 }
272
273 final FessConfig fessConfig = ComponentUtil.getFessConfig();
274 final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
275 final String sessionId = crawlingInfoHelper.getCanonicalSessionId(responseData.getSessionId());
276 final PathMappingHelper pathMappingHelper = ComponentUtil.getPathMappingHelper();
277 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
278 final CrawlingConfig crawlingConfig = crawlingConfigHelper.get(responseData.getSessionId());
279 final Date documentExpires = crawlingInfoHelper.getDocumentExpires(crawlingConfig);
280 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
281 final FileTypeHelper fileTypeHelper = ComponentUtil.getFileTypeHelper();
282 final DocumentHelper documentHelper = ComponentUtil.getDocumentHelper();
283 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
284 String url = responseData.getUrl();
285 final String indexingTarget = crawlingConfig.getIndexingTarget(url);
286 url = pathMappingHelper.replaceUrl(sessionId, url);
287 final String mimeType = responseData.getMimeType();
288
289 final Map<String, String> fieldConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.FIELD);
290 final Map<String, String> xpathConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.XPATH);
291
292 String urlEncoding;
293 final UrlQueue<?> urlQueue = CrawlingParameterUtil.getUrlQueue();
294 if (urlQueue != null && urlQueue.getEncoding() != null) {
295 urlEncoding = urlQueue.getEncoding();
296 } else {
297 urlEncoding = responseData.getCharSet();
298 }
299
300
301 final String configId = crawlingConfig.getConfigId();
302 if (configId != null) {
303 putResultDataBody(dataMap, fessConfig.getIndexFieldConfigId(), configId);
304 }
305
306 if (documentExpires != null) {
307 putResultDataBody(dataMap, fessConfig.getIndexFieldExpires(), documentExpires);
308 }
309
310 final String lang = systemHelper.normalizeLang(getSingleNodeValue(document, getLangXpath(fessConfig, xpathConfigMap), true));
311 if (lang != null) {
312 putResultDataBody(dataMap, fessConfig.getIndexFieldLang(), lang);
313 }
314
315
316 final String body = getSingleNodeValue(document, getContentXpath(fessConfig, xpathConfigMap), prunedContent);
317 putResultDataBody(dataMap, fessConfig.getIndexFieldContent(), documentHelper.getContent(responseData, body, dataMap));
318 if ((Constants.TRUE.equalsIgnoreCase(fieldConfigMap.get(fessConfig.getIndexFieldCache())) || fessConfig
319 .isCrawlerDocumentCacheEnabled()) && fessConfig.isSupportedDocumentCacheMimetypes(mimeType)) {
320 if (responseData.getContentLength() > 0
321 && responseData.getContentLength() <= fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger().longValue()) {
322 String charSet = responseData.getCharSet();
323 if (charSet == null) {
324 charSet = Constants.UTF_8;
325 }
326 try (final BufferedInputStream is = new BufferedInputStream(responseData.getResponseBody())) {
327
328 putResultDataBody(dataMap, fessConfig.getIndexFieldCache(), new String(InputStreamUtil.getBytes(is), charSet));
329 putResultDataBody(dataMap, fessConfig.getIndexFieldHasCache(), Constants.TRUE);
330 } catch (final Exception e) {
331 logger.warn("Failed to write a cache: " + sessionId + ":" + responseData, e);
332 }
333 } else {
334 logger.debug("Content size is too large({} > {}): {}", responseData.getContentLength(),
335 fessConfig.getCrawlerDocumentCacheMaxSizeAsInteger(), responseData.getUrl());
336 }
337 }
338
339 final String digest = getSingleNodeValue(document, getDigestXpath(fessConfig, xpathConfigMap), false);
340 if (StringUtil.isNotBlank(digest)) {
341 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(), digest);
342 } else {
343 putResultDataBody(dataMap, fessConfig.getIndexFieldDigest(),
344 documentHelper.getDigest(responseData, body, dataMap, fessConfig.getCrawlerDocumentHtmlMaxDigestLengthAsInteger()));
345 }
346
347 putResultDataBody(dataMap, fessConfig.getIndexFieldSegment(), sessionId);
348
349 putResultDataBody(dataMap, fessConfig.getIndexFieldHost(), getHost(url));
350
351 putResultDataBody(dataMap, fessConfig.getIndexFieldSite(), getSite(url, urlEncoding));
352
353 final String fileName = getFileName(url, urlEncoding);
354 if (StringUtil.isNotBlank(fileName)) {
355 putResultDataBody(dataMap, fessConfig.getIndexFieldFilename(), fileName);
356 }
357
358 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
359
360 final Date now = systemHelper.getCurrentTime();
361 putResultDataBody(dataMap, fessConfig.getIndexFieldCreated(), now);
362
363 putResultDataBody(dataMap, fessConfig.getIndexFieldAnchor(), getAnchorList(document, responseData));
364
365 putResultDataBody(dataMap, fessConfig.getIndexFieldMimetype(), mimeType);
366 if (fileTypeHelper != null) {
367
368 putResultDataBody(dataMap, fessConfig.getIndexFieldFiletype(), fileTypeHelper.get(mimeType));
369 }
370
371 putResultDataBody(dataMap, fessConfig.getIndexFieldContentLength(), Long.toString(responseData.getContentLength()));
372
373 final Date lastModified = responseData.getLastModified();
374 if (lastModified != null) {
375 putResultDataBody(dataMap, fessConfig.getIndexFieldLastModified(), lastModified);
376
377 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), lastModified);
378 } else {
379
380 putResultDataBody(dataMap, fessConfig.getIndexFieldTimestamp(), now);
381 }
382
383 putResultDataBody(dataMap, Constants.INDEXING_TARGET, indexingTarget);
384
385 putResultDataBody(dataMap, fessConfig.getIndexFieldBoost(), crawlingConfig.getDocumentBoost());
386
387 final Set<String> labelTypeSet = new HashSet<>();
388 for (final String labelType : crawlingConfig.getLabelTypeValues()) {
389 labelTypeSet.add(labelType);
390 }
391 labelTypeSet.addAll(labelTypeHelper.getMatchedLabelValueSet(url));
392 putResultDataBody(dataMap, fessConfig.getIndexFieldLabel(), labelTypeSet);
393
394 final List<String> roleTypeList = new ArrayList<>();
395 stream(crawlingConfig.getPermissions()).of(stream -> stream.forEach(p -> roleTypeList.add(p)));
396 putResultDataBody(dataMap, fessConfig.getIndexFieldRole(), roleTypeList);
397
398 putResultDataBody(dataMap, fessConfig.getIndexFieldVirtualHost(),
399 stream(crawlingConfig.getVirtualHosts()).get(stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n])));
400
401 putResultDataBody(dataMap, fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
402
403 String parentUrl = responseData.getParentUrl();
404 if (StringUtil.isNotBlank(parentUrl)) {
405 parentUrl = pathMappingHelper.replaceUrl(sessionId, parentUrl);
406 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), parentUrl);
407 putResultDataBody(dataMap, fessConfig.getIndexFieldParentId(), crawlingInfoHelper.generateId(dataMap));
408 putResultDataBody(dataMap, fessConfig.getIndexFieldUrl(), url);
409 }
410
411 final String thumbnailUrl = getThumbnailUrl(responseData, document);
412 if (StringUtil.isNotBlank(thumbnailUrl)) {
413 putResultDataBody(dataMap, fessConfig.getIndexFieldThumbnail(), thumbnailUrl);
414 }
415
416
417 final Map<String, String> scriptConfigMap = crawlingConfig.getConfigParameterMap(ConfigName.SCRIPT);
418 xpathConfigMap.entrySet().stream().filter(e -> !e.getKey().startsWith("default.")).forEach(e -> {
419 final String key = e.getKey();
420 final String value = getSingleNodeValue(document, e.getValue(), true);
421 putResultDataWithTemplate(dataMap, key, value, scriptConfigMap.get(key));
422 });
423 crawlingConfig.getConfigParameterMap(ConfigName.VALUE).entrySet().stream().forEach(e -> {
424 final String key = e.getKey();
425 final String value = e.getValue();
426 putResultDataWithTemplate(dataMap, key, value, scriptConfigMap.get(key));
427 });
428 }
429
430 protected String getLangXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
431 final String xpath = xpathConfigMap.get("default.lang");
432 if (StringUtil.isNotBlank(xpath)) {
433 return xpath;
434 }
435 return fessConfig.getCrawlerDocumentHtmlLangXpath();
436 }
437
438 protected String getContentXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
439 final String xpath = xpathConfigMap.get("default.content");
440 if (StringUtil.isNotBlank(xpath)) {
441 return xpath;
442 }
443 return fessConfig.getCrawlerDocumentHtmlContentXpath();
444 }
445
446 protected String getDigestXpath(final FessConfig fessConfig, final Map<String, String> xpathConfigMap) {
447 final String xpath = xpathConfigMap.get("default.digest");
448 if (StringUtil.isNotBlank(xpath)) {
449 return xpath;
450 }
451 return fessConfig.getCrawlerDocumentHtmlDigestXpath();
452 }
453
454 protected String getCanonicalUrl(final ResponseData responseData, final Document document) {
455 final String canonicalUrl = getSingleNodeValue(document, fessConfig.getCrawlerDocumentHtmlCanonicalXpath(), false);
456 if (StringUtil.isBlank(canonicalUrl)) {
457 return null;
458 }
459 return normalizeCanonicalUrl(responseData.getUrl(), canonicalUrl);
460 }
461
462 protected String normalizeCanonicalUrl(final String baseUrl, final String canonicalUrl) {
463 try {
464 final URL u = new URL(baseUrl);
465 return new URL(u, canonicalUrl.startsWith(":") ? u.getProtocol() + canonicalUrl : canonicalUrl).toString();
466 } catch (final MalformedURLException e) {
467 logger.warn("Invalid canonical url: " + baseUrl + " : " + canonicalUrl, e);
468 }
469 return null;
470 }
471
472 protected String removeCommentTag(final String content) {
473 if (content == null) {
474 return StringUtil.EMPTY;
475 }
476 String value = content;
477 int pos = value.indexOf("<!--");
478 while (pos >= 0) {
479 final int lastPos = value.indexOf("-->", pos);
480 if (lastPos >= 0) {
481 if (pos == 0) {
482 value = " " + value.substring(lastPos + 3);
483 } else {
484 value = value.substring(0, pos) + " " + value.substring(lastPos + 3);
485 }
486 } else {
487 break;
488 }
489 pos = value.indexOf("<!--");
490 }
491 return value;
492 }
493
494 protected String getSingleNodeValue(final Document document, final String xpath, final boolean pruned) {
495 StringBuilder buf = null;
496 NodeList list = null;
497 try {
498 list = getXPathAPI().selectNodeList(document, xpath);
499 for (int i = 0; i < list.getLength(); i++) {
500 if (buf == null) {
501 buf = new StringBuilder(1000);
502 }
503 Node node = list.item(i).cloneNode(true);
504 if (useGoogleOffOn) {
505 node = processGoogleOffOn(node, new ValueHolder<>(true));
506 }
507 if (pruned) {
508 node = pruneNode(node);
509 }
510 parseTextContent(node, buf);
511 }
512 } catch (final Exception e) {
513 logger.warn("Could not parse a value of " + xpath);
514 }
515 if (buf == null) {
516 return null;
517 }
518 return buf.toString().trim();
519 }
520
521 protected void parseTextContent(final Node node, final StringBuilder buf) {
522 if (node.hasChildNodes()) {
523 final NodeList nodeList = node.getChildNodes();
524 for (int i = 0; i < nodeList.getLength(); i++) {
525 final Node childNode = nodeList.item(i);
526 parseTextContent(childNode, buf);
527 }
528 } else if (node.getNodeType() == Node.TEXT_NODE) {
529 final String value = node.getTextContent();
530 if (value != null) {
531 final String content = value.trim();
532 if (content.length() > 0) {
533 buf.append(' ').append(content);
534 }
535 }
536 }
537 }
538
539 protected Node processGoogleOffOn(final Node node, final ValueHolder<Boolean> flag) {
540 final NodeList nodeList = node.getChildNodes();
541 List<Node> removedNodeList = null;
542 for (int i = 0; i < nodeList.getLength(); i++) {
543 final Node childNode = nodeList.item(i);
544 if (childNode.getNodeType() == Node.COMMENT_NODE) {
545 final String comment = childNode.getNodeValue().trim();
546 if (comment.startsWith("googleoff:")) {
547 flag.setValue(false);
548 } else if (comment.startsWith("googleon:")) {
549 flag.setValue(true);
550 }
551 }
552
553 if (!flag.getValue() && childNode.getNodeType() == Node.TEXT_NODE) {
554 if (removedNodeList == null) {
555 removedNodeList = new ArrayList<>();
556 }
557 removedNodeList.add(childNode);
558 } else {
559 processGoogleOffOn(childNode, flag);
560 }
561 }
562
563 if (removedNodeList != null) {
564 removedNodeList.stream().forEach(n -> node.removeChild(n));
565 }
566
567 return node;
568 }
569
570 protected Node pruneNode(final Node node) {
571 final NodeList nodeList = node.getChildNodes();
572 final List<Node> childNodeList = new ArrayList<>();
573 final List<Node> removedNodeList = new ArrayList<>();
574 for (int i = 0; i < nodeList.getLength(); i++) {
575 final Node childNode = nodeList.item(i);
576 if (isPrunedTag(childNode)) {
577 removedNodeList.add(childNode);
578 } else {
579 childNodeList.add(childNode);
580 }
581 }
582
583 for (final Node childNode : removedNodeList) {
584 node.removeChild(childNode);
585 }
586
587 for (final Node childNode : childNodeList) {
588 pruneNode(childNode);
589 }
590
591 return node;
592 }
593
594 protected boolean isPrunedTag(final Node node) {
595 for (final PrunedTag prunedTag : fessConfig.getCrawlerDocumentHtmlPrunedTagsAsArray()) {
596 if (prunedTag.matches(node)) {
597 return true;
598 }
599 }
600 return false;
601 }
602
603 protected String getMultipleNodeValue(final Document document, final String xpath) {
604 NodeList nodeList = null;
605 final StringBuilder buf = new StringBuilder(100);
606 try {
607 nodeList = getXPathAPI().selectNodeList(document, xpath);
608 for (int i = 0; i < nodeList.getLength(); i++) {
609 final Node node = nodeList.item(i);
610 buf.append(node.getTextContent());
611 buf.append("\n");
612 }
613 } catch (final Exception e) {
614 logger.warn("Could not parse a value of " + xpath, e);
615 }
616 return buf.toString().trim();
617 }
618
619 protected String replaceDuplicateHost(final String url) {
620 try {
621
622 final DuplicateHostHelper duplicateHostHelper = ComponentUtil.getDuplicateHostHelper();
623 return duplicateHostHelper.convert(url);
624 } catch (final Exception e) {
625 return url;
626 }
627 }
628
629 protected List<String> getAnchorList(final Document document, final ResponseData responseData) {
630 List<RequestData> anchorList = new ArrayList<>();
631 final String baseHref = getBaseHref(document);
632 try {
633 final URL url = getBaseUrl(responseData.getUrl(), baseHref);
634 for (final Map.Entry<String, String> entry : childUrlRuleMap.entrySet()) {
635 for (final String u : getUrlFromTagAttribute(url, document, entry.getKey(), entry.getValue(), responseData.getCharSet())) {
636 anchorList.add(RequestDataBuilder.newRequestData().get().url(u).build());
637 }
638 }
639 anchorList = convertChildUrlList(anchorList);
640 } catch (final Exception e) {
641 logger.warn("Could not parse anchor tags.", e);
642 }
643
644 final List<String> urlList = new ArrayList<>(anchorList.size());
645 for (final RequestData requestData : anchorList) {
646 urlList.add(requestData.getUrl());
647 }
648 return urlList;
649 }
650
651 protected URL getBaseUrl(final String currentUrl, final String baseHref) throws MalformedURLException {
652 if (baseHref != null) {
653 return getURL(currentUrl, baseHref);
654 }
655 return new URL(currentUrl);
656 }
657
658 @Override
659 protected List<RequestData> convertChildUrlList(final List<RequestData> urlList) {
660 if (urlList != null) {
661 for (final RequestData requestData : urlList) {
662 String url = requestData.getUrl();
663 for (final Map.Entry<String, String> entry : convertUrlMap.entrySet()) {
664 url = url.replaceAll(entry.getKey(), entry.getValue());
665 }
666 requestData.setUrl(replaceDuplicateHost(url));
667 }
668 }
669 return urlList;
670 }
671
672 @Override
673 public Object getData(final AccessResultData<?> accessResultData) {
674 final byte[] data = accessResultData.getData();
675 if (data != null) {
676 try {
677 return SerializeUtil.fromBinaryToObject(data);
678 } catch (final Exception e) {
679 throw new CrawlerSystemException("Could not create an instanced from bytes.", e);
680 }
681 }
682 return new HashMap<String, Object>();
683 }
684
685 @Override
686 protected void addChildUrlFromTagAttribute(final List<String> urlList, final URL url, final String attrValue, final String encoding) {
687 final String urlValue = attrValue.trim();
688 URL childUrl;
689 String u = null;
690 try {
691 childUrl = new URL(url, urlValue.startsWith(":") ? url.getProtocol() + urlValue : urlValue);
692 u = encodeUrl(normalizeUrl(childUrl.toExternalForm()), encoding);
693 } catch (final MalformedURLException e) {
694 final int pos = urlValue.indexOf(':');
695 if (pos > 0 && pos < 10) {
696 u = encodeUrl(normalizeUrl(urlValue), encoding);
697 }
698 }
699
700 if (u == null) {
701 logger.warn("Ignored child URL: " + attrValue + " in " + url);
702 return;
703 }
704
705 if (logger.isDebugEnabled()) {
706 logger.debug(attrValue + " -> " + u);
707 }
708 if (StringUtil.isNotBlank(u)) {
709 if (logger.isDebugEnabled()) {
710 logger.debug("Add Child: " + u);
711 }
712 urlList.add(u);
713 } else {
714 if (logger.isDebugEnabled()) {
715 logger.debug("Skip Child: " + u);
716 }
717 }
718 }
719
720 private boolean isUtf8BomBytes(final byte[] b) {
721 return b[0] == (byte) 0xEF && b[1] == (byte) 0xBB && b[2] == (byte) 0xBF;
722 }
723
724 public void setUseGoogleOffOn(final boolean useGoogleOffOn) {
725 this.useGoogleOffOn = useGoogleOffOn;
726 }
727
728 protected String getThumbnailUrl(final ResponseData responseData, final Document document) {
729
730 try {
731
732 final Node thumbnailNode = getXPathAPI().selectSingleNode(document, META_NAME_THUMBNAIL_CONTENT);
733 if (thumbnailNode != null) {
734 final String content = thumbnailNode.getTextContent();
735 if (StringUtil.isNotBlank(content)) {
736 final URL thumbnailUrl = getURL(responseData.getUrl(), content);
737 if (thumbnailUrl != null) {
738 return thumbnailUrl.toExternalForm();
739 }
740 }
741 }
742
743
744 final Node ogImageNode = getXPathAPI().selectSingleNode(document, META_PROPERTY_OGIMAGE_CONTENT);
745 if (ogImageNode != null) {
746 final String content = ogImageNode.getTextContent();
747 if (StringUtil.isNotBlank(content)) {
748 final URL thumbnailUrl = getURL(responseData.getUrl(), content);
749 if (thumbnailUrl != null) {
750 return thumbnailUrl.toExternalForm();
751 }
752 }
753 }
754
755 final NodeList imgNodeList = getXPathAPI().selectNodeList(document, fessConfig.getThumbnailHtmlImageXpath());
756 String firstThumbnailUrl = null;
757 for (int i = 0; i < imgNodeList.getLength(); i++) {
758 final Node imgNode = imgNodeList.item(i);
759 if (logger.isDebugEnabled()) {
760 logger.debug("img tag: " + imgNode);
761 }
762 final NamedNodeMap attributes = imgNode.getAttributes();
763 final String thumbnailUrl = getThumbnailSrc(responseData.getUrl(), attributes);
764 final Integer height = getAttributeAsInteger(attributes, "height");
765 final Integer width = getAttributeAsInteger(attributes, "width");
766 if (!fessConfig.isThumbnailHtmlImageUrl(thumbnailUrl)) {
767 continue;
768 } else if (height != null && width != null) {
769 try {
770 if (fessConfig.validateThumbnailSize(width, height)) {
771 return thumbnailUrl;
772 }
773 } catch (final Exception e) {
774 logger.debug("Failed to parse " + imgNode + " at " + responseData.getUrl(), e);
775 }
776 } else if (firstThumbnailUrl == null) {
777 firstThumbnailUrl = thumbnailUrl;
778 }
779 }
780
781 if (firstThumbnailUrl != null) {
782 return firstThumbnailUrl;
783 }
784 } catch (final Exception e) {
785 logger.warn("Failed to retrieve thumbnail url from " + responseData.getUrl(), e);
786 }
787 return null;
788 }
789
790 protected String getThumbnailSrc(final String url, final NamedNodeMap attributes) {
791 final Node srcNode = attributes.getNamedItem("src");
792 if (srcNode != null) {
793 try {
794 final URL thumbnailUrl = getURL(url, srcNode.getTextContent());
795 if (thumbnailUrl != null) {
796 return thumbnailUrl.toExternalForm();
797 }
798 } catch (final Exception e) {
799 if (logger.isDebugEnabled()) {
800 logger.debug("Failed to parse thumbnail url for " + url + " : " + attributes, e);
801 }
802 }
803 }
804 return null;
805 }
806
807 protected Integer getAttributeAsInteger(final NamedNodeMap attributes, final String name) {
808 final Node namedItem = attributes.getNamedItem(name);
809 if (namedItem == null) {
810 return null;
811 }
812 final String value = namedItem.getTextContent();
813 if (value == null) {
814 return null;
815 }
816 try {
817 return Integer.parseInt(value);
818 } catch (final NumberFormatException e) {
819 if (value.endsWith("%") || value.endsWith("px")) {
820 return null;
821 }
822 return 0;
823 }
824 }
825
826 protected URL getURL(final String currentUrl, final String url) throws MalformedURLException {
827 if (url != null) {
828 if (url.startsWith("://")) {
829 final String protocol = currentUrl.split(":")[0];
830 return new URL(protocol + url);
831 } else if (url.startsWith("//")) {
832 final String protocol = currentUrl.split(":")[0];
833 return new URL(protocol + ":" + url);
834 } else if (url.startsWith("/") || url.indexOf(':') == -1) {
835 return new URL(new URL(currentUrl), url);
836 }
837 return new URL(url);
838 }
839 return null;
840 }
841 }