Package org.codelibs.fess.crawler.transformer

Class FessXpathTransformer

java.lang.Object

org.codelibs.fess.crawler.transformer.impl.AbstractTransformer

org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

org.codelibs.fess.crawler.transformer.impl.XpathTransformer

org.codelibs.fess.crawler.transformer.FessXpathTransformer

All Implemented Interfaces:

FessTransformer, org.codelibs.fess.crawler.transformer.Transformer

public class FessXpathTransformer
extends org.codelibs.fess.crawler.transformer.impl.XpathTransformer
implements FessTransformer

Field Summary

Fields

Modifier and Type	Field	Description
protected Map<String,String>	convertUrlMap
protected FessConfig	fessConfig
protected Map<String,Boolean>	fieldPrunedRuleMap
boolean	prunedContent
protected boolean	useGoogleOffOn

Fields inherited from class org.codelibs.fess.crawler.transformer.impl.XpathTransformer

charsetName, dataClass, fieldRuleMap, trimSpaceEnabled

Fields inherited from class org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

childUrlRuleMap, crawlerContainer, defaultEncoding, featureMap, invalidUrlPattern, LOCATION_HEADER, preloadSizeForCharset, propertyMap

Fields inherited from class org.codelibs.fess.crawler.transformer.impl.AbstractTransformer

name

Fields inherited from interface org.codelibs.fess.crawler.transformer.FessTransformer

parentEncodingMap

Constructor Summary

Constructors

Constructor	Description
FessXpathTransformer()

Method Summary

Modifier and Type	Method	Description
protected void	addChildUrlFromTagAttribute(List<String> urlList, URL url, String attrValue, String encoding)
void	addConvertUrl(String regex, String replacement)
void	addFieldRule(String name, String xpath, boolean isPruned)
protected List<org.codelibs.fess.crawler.entity.RequestData>	convertChildUrlList(List<org.codelibs.fess.crawler.entity.RequestData> urlList)
protected List<String>	getAnchorList(Document document, org.codelibs.fess.crawler.entity.ResponseData responseData)
protected Integer	getAttributeAsInteger(NamedNodeMap attributes, String name)
protected URL	getBaseUrl(String currentUrl, String baseHref)
protected String	getCanonicalUrl(org.codelibs.fess.crawler.entity.ResponseData responseData, Document document)
protected Map<String,String>	getConfigPrameterMap(org.codelibs.fess.crawler.entity.ResponseData responseData, CrawlingConfig.ConfigName config)
protected String	getContentXpath(FessConfig fessConfig, Map<String,String> xpathConfigMap)
Object	getData(org.codelibs.fess.crawler.entity.AccessResultData<?> accessResultData)
protected String	getDigestXpath(FessConfig fessConfig, Map<String,String> xpathConfigMap)
FessConfig	getFessConfig()
protected String	getLangXpath(FessConfig fessConfig, Map<String,String> xpathConfigMap)
org.apache.logging.log4j.Logger	getLogger()
protected String	getMultipleNodeValue(Document document, String xpath)
protected PathMappingHelper	getPathMappingHelper()
protected String	getSingleNodeValue(Document document, String xpath, boolean pruned)
protected String	getThumbnailSrc(String url, NamedNodeMap attributes)
protected String	getThumbnailUrl(org.codelibs.fess.crawler.entity.ResponseData responseData, Document document)
protected URL	getURL(String currentUrl, String url)
void	init()
protected boolean	isPrunedTag(Node node)
protected boolean	isValidCanonicalUrl(String url, String canonicalUrl)
protected boolean	isValidUrl(String urlStr)
protected String	normalizeCanonicalUrl(String baseUrl, String canonicalUrl)
protected void	normalizeData(org.codelibs.fess.crawler.entity.ResponseData responseData, Map<String,Object> dataMap)
protected void	parseTextContent(Node node, StringBuilder buf)
protected Node	processGoogleOffOn(Node node, org.codelibs.core.misc.ValueHolder<Boolean> flag)
protected void	processMetaRobots(org.codelibs.fess.crawler.entity.ResponseData responseData, org.codelibs.fess.crawler.entity.ResultData resultData, Document document)
protected void	processXRobotsTag(org.codelibs.fess.crawler.entity.ResponseData responseData, org.codelibs.fess.crawler.entity.ResultData resultData)
protected Node	pruneNode(Node node)
protected void	putAdditionalData(Map<String,Object> dataMap, org.codelibs.fess.crawler.entity.ResponseData responseData, Document document)
protected String	removeCommentTag(String content)
protected String	replaceDuplicateHost(String url)
void	setConvertUrlMap(Map<String,String> convertUrlMap)
void	setUseGoogleOffOn(boolean useGoogleOffOn)
protected void	storeData(org.codelibs.fess.crawler.entity.ResponseData responseData, org.codelibs.fess.crawler.entity.ResultData resultData)

Methods inherited from class org.codelibs.fess.crawler.transformer.impl.XpathTransformer

addFieldRule, getAdditionalData, getCharsetName, getDataClass, getFieldRuleMap, getResultDataBody, getResultDataBody, getResultDataFooter, getResultDataHeader, isTrimSpace, setCharsetName, setDataClass, setFieldRuleMap, setTrimSpace, trimSpace

Methods inherited from class org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

addChildUrlRule, addFeature, addProperty, encodeUrl, getBaseHref, getChildUrlRuleMap, getDefaultEncoding, getDomParser, getDuplicateUrl, getFeatureMap, getInvalidUrlPattern, getPreloadSizeForCharset, getPropertyMap, getUrlFromTagAttribute, getXPathAPI, isHtml, isSupportedCharset, isValidPath, loadCharset, normalizeEncoding, normalizeUrl, parseCharset, setChildUrlRuleMap, setDefaultEncoding, setFeatureMap, setInvalidUrlPattern, setPreloadSizeForCharset, setPropertyMap, storeChildUrls, transform, updateCharset

Methods inherited from class org.codelibs.fess.crawler.transformer.impl.AbstractTransformer

getName, setName

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.codelibs.fess.crawler.transformer.FessTransformer

abbreviateSite, decodeUrlAsName, evaluateValue, getFileName, getHost, getMaxSiteLength, getParentEncoding, getSite, putResultDataBody, putResultDataWithTemplate

Field Detail

prunedContent

public boolean prunedContent

convertUrlMap

protected Map<String,String> convertUrlMap

fessConfig

protected FessConfig fessConfig

useGoogleOffOn

protected boolean useGoogleOffOn

fieldPrunedRuleMap

protected Map<String,Boolean> fieldPrunedRuleMap

Constructor Detail

FessXpathTransformer

public FessXpathTransformer()

Method Detail

init

@PostConstruct
public void init()

getFessConfig

public FessConfig getFessConfig()

Specified by:

getFessConfig in interface FessTransformer

getLogger

public org.apache.logging.log4j.Logger getLogger()

Specified by:

getLogger in interface FessTransformer

storeData

protected void storeData(org.codelibs.fess.crawler.entity.ResponseData responseData,
                         org.codelibs.fess.crawler.entity.ResultData resultData)

Overrides:

storeData in class org.codelibs.fess.crawler.transformer.impl.XpathTransformer

normalizeData

protected void normalizeData(org.codelibs.fess.crawler.entity.ResponseData responseData,
                             Map<String,Object> dataMap)

processMetaRobots

protected void processMetaRobots(org.codelibs.fess.crawler.entity.ResponseData responseData,
                                 org.codelibs.fess.crawler.entity.ResultData resultData,
                                 Document document)

processXRobotsTag

protected void processXRobotsTag(org.codelibs.fess.crawler.entity.ResponseData responseData,
                                 org.codelibs.fess.crawler.entity.ResultData resultData)

getConfigPrameterMap

protected Map<String,String> getConfigPrameterMap(org.codelibs.fess.crawler.entity.ResponseData responseData,
                                                        CrawlingConfig.ConfigName config)

isValidUrl

protected boolean isValidUrl(String urlStr)

isValidCanonicalUrl

protected boolean isValidCanonicalUrl(String url,
                                      String canonicalUrl)

putAdditionalData

protected void putAdditionalData(Map<String,Object> dataMap,
                                 org.codelibs.fess.crawler.entity.ResponseData responseData,
                                 Document document)

getLangXpath

protected String getLangXpath(FessConfig fessConfig,
                              Map<String,String> xpathConfigMap)

getContentXpath

protected String getContentXpath(FessConfig fessConfig,
                                 Map<String,String> xpathConfigMap)

getDigestXpath

protected String getDigestXpath(FessConfig fessConfig,
                                Map<String,String> xpathConfigMap)

getCanonicalUrl

protected String getCanonicalUrl(org.codelibs.fess.crawler.entity.ResponseData responseData,
                                 Document document)

normalizeCanonicalUrl

protected String normalizeCanonicalUrl(String baseUrl,
                                       String canonicalUrl)

removeCommentTag

protected String removeCommentTag(String content)

getSingleNodeValue

protected String getSingleNodeValue(Document document,
                                    String xpath,
                                    boolean pruned)

parseTextContent

protected void parseTextContent(Node node,
                                StringBuilder buf)

processGoogleOffOn

protected Node processGoogleOffOn(Node node,
                                  org.codelibs.core.misc.ValueHolder<Boolean> flag)

pruneNode

protected Node pruneNode(Node node)

isPrunedTag

protected boolean isPrunedTag(Node node)

getMultipleNodeValue

protected String getMultipleNodeValue(Document document,
                                      String xpath)

replaceDuplicateHost

protected String replaceDuplicateHost(String url)

getAnchorList

protected List<String> getAnchorList(Document document,
                                     org.codelibs.fess.crawler.entity.ResponseData responseData)

getBaseUrl

protected URL getBaseUrl(String currentUrl,
                         String baseHref)
                  throws MalformedURLException

Throws:

MalformedURLException

convertChildUrlList

protected List<org.codelibs.fess.crawler.entity.RequestData> convertChildUrlList(List<org.codelibs.fess.crawler.entity.RequestData> urlList)

Overrides:

convertChildUrlList in class org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

getPathMappingHelper

protected PathMappingHelper getPathMappingHelper()

getData

public Object getData(org.codelibs.fess.crawler.entity.AccessResultData<?> accessResultData)

Specified by:

getData in interface org.codelibs.fess.crawler.transformer.Transformer

Overrides:

getData in class org.codelibs.fess.crawler.transformer.impl.XpathTransformer

addChildUrlFromTagAttribute

protected void addChildUrlFromTagAttribute(List<String> urlList,
                                           URL url,
                                           String attrValue,
                                           String encoding)

Overrides:

addChildUrlFromTagAttribute in class org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

setUseGoogleOffOn

public void setUseGoogleOffOn(boolean useGoogleOffOn)

getThumbnailUrl

protected String getThumbnailUrl(org.codelibs.fess.crawler.entity.ResponseData responseData,
                                 Document document)

getThumbnailSrc

protected String getThumbnailSrc(String url,
                                 NamedNodeMap attributes)

getAttributeAsInteger

protected Integer getAttributeAsInteger(NamedNodeMap attributes,
                                        String name)

getURL

protected URL getURL(String currentUrl,
                     String url)
              throws MalformedURLException

Throws:

MalformedURLException

addFieldRule

public void addFieldRule(String name,
                         String xpath,
                         boolean isPruned)

setConvertUrlMap

public void setConvertUrlMap(Map<String,String> convertUrlMap)

addConvertUrl

public void addConvertUrl(String regex,
                          String replacement)