1 /*
2 * Copyright 2012-2025 CodeLibs Project and the Others.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13 * either express or implied. See the License for the specific language
14 * governing permissions and limitations under the License.
15 */
16 package org.codelibs.fess.util;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.util.Objects;
21 import java.util.regex.Matcher;
22 import java.util.regex.Pattern;
23
24 import org.apache.commons.lang3.StringUtils;
25 import org.codelibs.core.lang.StringUtil;
26 import org.codelibs.core.stream.StreamUtil;
27 import org.codelibs.fess.exception.FessSystemException;
28 import org.w3c.dom.Node;
29
30 /**
31 * Represents a tag configuration for pruning HTML content during document processing.
32 * This class defines tag patterns that match HTML elements based on tag name, CSS class, ID, or custom attributes.
33 * It is used to identify and remove unwanted HTML elements from crawled documents.
34 */
35 public class PrunedTag {
36 /** The HTML tag name to match (e.g., "div", "span", "p") */
37 private final String tag;
38 /** The ID attribute value to match */
39 private String id;
40 /** The CSS class name to match */
41 private String css;
42 /** The custom attribute name to match */
43 private String attrName;
44 /** The custom attribute value to match */
45 private String attrValue;
46
47 /**
48 * Creates a new PrunedTag instance with the specified tag name.
49 *
50 * @param tag the HTML tag name to match (e.g., "div", "span", "p")
51 */
52 public PrunedTag(final String tag) {
53 this.tag = tag;
54 }
55
56 /**
57 * Checks if this pruned tag configuration matches the given DOM node.
58 * The matching is based on tag name, and optionally ID, CSS class, or custom attributes.
59 *
60 * @param node the DOM node to check against this pruned tag configuration
61 * @return true if the node matches this pruned tag configuration, false otherwise
62 */
63 public boolean matches(final Node node) {
64 if (tag.equalsIgnoreCase(node.getNodeName())) {
65 if (attrName != null) {
66 final Node attr = node.getAttributes().getNamedItem(attrName);
67 if (attr == null || !attrValue.equals(attr.getNodeValue())) {
68 return false;
69 }
70 }
71 if (id == null) {
72 if (css == null) {
73 return true;
74 }
75 final Node classAttr = node.getAttributes().getNamedItem("class");
76 if (classAttr != null) {
77 final String value = classAttr.getNodeValue();
78 if (StringUtil.isNotBlank(value)) {
79 return StreamUtil.split(value, " ").get(stream -> stream.anyMatch(s -> css.equals(s)));
80 }
81 }
82 } else {
83 final Node idAttr = node.getAttributes().getNamedItem("id");
84 if (idAttr != null) {
85 final String value = idAttr.getNodeValue();
86 return id.equals(value);
87 }
88 }
89 }
90 return false;
91 }
92
93 @Override
94 public int hashCode() {
95 return Objects.hash(css, id, tag);
96 }
97
98 @Override
99 public boolean equals(final Object obj) {
100 if (this == obj) {
101 return true;
102 }
103 if (obj == null || getClass() != obj.getClass()) {
104 return false;
105 }
106 final PrunedTag other = (PrunedTag) obj;
107 return StringUtils.compare(tag, other.tag) == 0 //
108 && StringUtils.compare(css, other.css) == 0 //
109 && StringUtils.compare(id, other.id) == 0 //
110 && StringUtils.compare(attrName, other.attrName) == 0 //
111 && StringUtils.compare(attrValue, other.attrValue) == 0;
112 }
113
114 /**
115 * Sets the ID attribute value that this pruned tag should match.
116 *
117 * @param id the ID attribute value to match
118 */
119 public void setId(final String id) {
120 this.id = id;
121 }
122
123 /**
124 * Sets the CSS class name that this pruned tag should match.
125 *
126 * @param css the CSS class name to match
127 */
128 public void setCss(final String css) {
129 this.css = css;
130 }
131
132 /**
133 * Sets a custom attribute name-value pair that this pruned tag should match.
134 *
135 * @param name the attribute name to match
136 * @param value the attribute value to match
137 */
138 public void setAttr(final String name, final String value) {
139 attrName = name;
140 attrValue = value;
141 }
142
143 @Override
144 public String toString() {
145 return "PrunedTag [tag=" + tag + ", id=" + id + ", css=" + css + ", attrName=" + attrName + ", attrValue=" + attrValue + "]";
146 }
147
148 /**
149 * Parses a comma-separated string of pruned tag configurations into an array of PrunedTag objects.
150 * Each tag configuration follows the pattern: tagname[attr=value].classname#id
151 *
152 * Examples:
153 * - "div.content" matches div elements with class "content"
154 * - "span#header" matches span elements with ID "header"
155 * - "p[data-type=ad]" matches p elements with data-type attribute equal to "ad"
156 *
157 * @param value the comma-separated string of pruned tag configurations
158 * @return an array of PrunedTag objects parsed from the input string
159 * @throws FessSystemException if the input string contains invalid tag patterns
160 */
161 public static PrunedTag[] parse(final String value) {
162 return split(value, ",").get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
163 final Pattern pattern = Pattern.compile("(\\w+)(\\[[^\\]]+\\])?(\\.[\\w\\-]+)?(#[\\w\\-]+)?");
164 final Matcher matcher = pattern.matcher(v.trim());
165 if (matcher.matches()) {
166 final PrunedTag tag = new PrunedTag(matcher.group(1));
167 if (matcher.group(2) != null) {
168 final String attrPair = matcher.group(2).substring(1, matcher.group(2).length() - 1);
169 final Matcher equalMatcher = Pattern.compile("([\\w\\-]+)=(\\S+)").matcher(attrPair);
170 if (equalMatcher.matches()) {
171 tag.setAttr(equalMatcher.group(1), equalMatcher.group(2));
172 }
173 }
174 if (matcher.group(3) != null) {
175 tag.setCss(matcher.group(3).substring(1));
176 }
177 if (matcher.group(4) != null) {
178 tag.setId(matcher.group(4).substring(1));
179 }
180 return tag;
181 }
182 throw new FessSystemException("Invalid pruned tag format: '" + v
183 + "'. Expected format: tagname[attr=value].classname#id (e.g., div.content, span#header, p[data-type=ad])");
184 }).toArray(n -> new PrunedTag[n]));
185 }
186 }