Package org.codelibs.fess.crawler.transformer

Class FessXpathTransformer

java.lang.Object
org.codelibs.fess.crawler.transformer.impl.AbstractTransformer
org.codelibs.fess.crawler.transformer.impl.HtmlTransformer
org.codelibs.fess.crawler.transformer.impl.XpathTransformer
org.codelibs.fess.crawler.transformer.FessXpathTransformer
All Implemented Interfaces:
FessTransformer, org.codelibs.fess.crawler.transformer.Transformer
public class FessXpathTransformer extends org.codelibs.fess.crawler.transformer.impl.XpathTransformer implements FessTransformer

  • Field Details

    • prunedContent

      public boolean prunedContent

    • convertUrlMap

      protected Map<String,String> convertUrlMap

    • fessConfig

      protected FessConfig fessConfig

    • useGoogleOffOn

      protected boolean useGoogleOffOn

    • fieldPrunedRuleMap

      protected Map<String,Boolean> fieldPrunedRuleMap

  • Constructor Details

    • FessXpathTransformer

      public FessXpathTransformer()

  • Method Details

    • init

      @PostConstruct public void init()

    • getFessConfig

      public FessConfig getFessConfig()
      Specified by:
      getFessConfig in interface FessTransformer

    • getLogger

      public org.apache.logging.log4j.Logger getLogger()
      Specified by:
      getLogger in interface FessTransformer

    • storeData

      protected void storeData(org.codelibs.fess.crawler.entity.ResponseData responseData, org.codelibs.fess.crawler.entity.ResultData resultData)
      Overrides:
      storeData in class org.codelibs.fess.crawler.transformer.impl.XpathTransformer

    • normalizeData

      protected void normalizeData(org.codelibs.fess.crawler.entity.ResponseData responseData, Map<String,Object> dataMap)

    • processMetaRobots

      protected void processMetaRobots(org.codelibs.fess.crawler.entity.ResponseData responseData, org.codelibs.fess.crawler.entity.ResultData resultData, Document document)

    • processXRobotsTag

      protected void processXRobotsTag(org.codelibs.fess.crawler.entity.ResponseData responseData, org.codelibs.fess.crawler.entity.ResultData resultData)

    • getConfigPrameterMap

      protected Map<String,String> getConfigPrameterMap(org.codelibs.fess.crawler.entity.ResponseData responseData, CrawlingConfig.ConfigName config)

    • isValidUrl

      protected boolean isValidUrl(String urlStr)

    • isValidCanonicalUrl

      protected boolean isValidCanonicalUrl(String url, String canonicalUrl)

    • putAdditionalData

      protected void putAdditionalData(Map<String,Object> dataMap, org.codelibs.fess.crawler.entity.ResponseData responseData, Document document)

    • getLangXpath

      protected String getLangXpath(FessConfig fessConfig, Map<String,String> xpathConfigMap)

    • getContentXpath

      protected String getContentXpath(FessConfig fessConfig, Map<String,String> xpathConfigMap)

    • getDigestXpath

      protected String getDigestXpath(FessConfig fessConfig, Map<String,String> xpathConfigMap)

    • getCanonicalUrl

      protected String getCanonicalUrl(org.codelibs.fess.crawler.entity.ResponseData responseData, Document document)

    • normalizeCanonicalUrl

      protected String normalizeCanonicalUrl(String baseUrl, String canonicalUrl)

    • removeCommentTag

      protected String removeCommentTag(String content)

    • getSingleNodeValue

      protected String getSingleNodeValue(Document document, String xpath, boolean pruned)

    • parseTextContent

      protected void parseTextContent(Node node, StringBuilder buf)

    • processGoogleOffOn

      protected Node processGoogleOffOn(Node node, org.codelibs.core.misc.ValueHolder<Boolean> flag)

    • pruneNode

      protected Node pruneNode(Node node)

    • isPrunedTag

      protected boolean isPrunedTag(Node node)

    • getMultipleNodeValue

      protected String getMultipleNodeValue(Document document, String xpath)

    • replaceDuplicateHost

      protected String replaceDuplicateHost(String url)

    • getAnchorList

      protected List<String> getAnchorList(Document document, org.codelibs.fess.crawler.entity.ResponseData responseData)

    • getBaseUrl

      protected URL getBaseUrl(String currentUrl, String baseHref) throws MalformedURLException
      Throws:
      MalformedURLException

    • convertChildUrlList

      protected List<org.codelibs.fess.crawler.entity.RequestData> convertChildUrlList(List<org.codelibs.fess.crawler.entity.RequestData> urlList)
      Overrides:
      convertChildUrlList in class org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

    • getPathMappingHelper

      protected PathMappingHelper getPathMappingHelper()

    • getData

      public Object getData(org.codelibs.fess.crawler.entity.AccessResultData<?> accessResultData)
      Specified by:
      getData in interface org.codelibs.fess.crawler.transformer.Transformer
      Overrides:
      getData in class org.codelibs.fess.crawler.transformer.impl.XpathTransformer

    • addChildUrlFromTagAttribute

      protected void addChildUrlFromTagAttribute(List<String> urlList, URL url, String attrValue, String encoding)
      Overrides:
      addChildUrlFromTagAttribute in class org.codelibs.fess.crawler.transformer.impl.HtmlTransformer

    • setUseGoogleOffOn

      public void setUseGoogleOffOn(boolean useGoogleOffOn)

    • getThumbnailUrl

      protected String getThumbnailUrl(org.codelibs.fess.crawler.entity.ResponseData responseData, Document document)

    • getThumbnailSrc

      protected String getThumbnailSrc(String url, NamedNodeMap attributes)

    • getAttributeAsInteger

      protected Integer getAttributeAsInteger(NamedNodeMap attributes, String name)

    • getURL

      protected URL getURL(String currentUrl, String url) throws MalformedURLException
      Throws:
      MalformedURLException

    • addFieldRule

      public void addFieldRule(String name, String xpath, boolean isPruned)

    • setConvertUrlMap

      public void setConvertUrlMap(Map<String,String> convertUrlMap)

    • addConvertUrl

      public void addConvertUrl(String regex, String replacement)