1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.util;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.HashMap;
23 import java.util.LinkedList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.regex.Pattern;
27 import java.util.stream.Collectors;
28
29 import javax.xml.XMLConstants;
30 import javax.xml.parsers.SAXParser;
31 import javax.xml.parsers.SAXParserFactory;
32
33 import org.apache.logging.log4j.LogManager;
34 import org.apache.logging.log4j.Logger;
35 import org.codelibs.core.lang.StringUtil;
36 import org.codelibs.fess.Constants;
37 import org.codelibs.fess.es.config.exentity.FileConfig;
38 import org.codelibs.fess.es.config.exentity.LabelType;
39 import org.codelibs.fess.es.config.exentity.WebConfig;
40 import org.codelibs.fess.exception.GsaConfigException;
41 import org.dbflute.optional.OptionalEntity;
42 import org.xml.sax.Attributes;
43 import org.xml.sax.InputSource;
44 import org.xml.sax.SAXException;
45 import org.xml.sax.helpers.DefaultHandler;
46
47 public class GsaConfigParser extends DefaultHandler {
48
49 private static final Logger logger = LogManager.getLogger(GsaConfigParser.class);
50
51 public static final String REGEXP = "regexp:";
52
53 public static final String REGEXP_CASE = "regexpCase:";
54
55 public static final String REGEXP_IGNORE_CASE = "regexpIgnoreCase:";
56
57 public static final String CONTAINS = "contains:";
58
59 protected static final String COLLECTIONS = "collections";
60
61 protected static final String COLLECTION = "collection";
62
63 protected static final String GLOBALPARAMS = "globalparams";
64
65 protected static final String START_URLS = "start_urls";
66
67 protected static final String GOOD_URLS = "good_urls";
68
69 protected static final String BAD_URLS = "bad_urls";
70
71 protected String[] webProtocols = new String[] { "http:", "https:" };
72
73 protected String[] fileProtocols = new String[] { "file:", "smb:", "smb1:", "ftp:", "storage:" };
74
75 protected LinkedList<String> tagQueue;
76
77 protected List<LabelType> labelList;
78
79 protected LabelType labelType;
80
81 protected Map<String, String> globalParams = new HashMap<>();
82
83 protected WebConfig webConfig = null;
84
85 protected FileConfig fileConfig = null;
86
87 protected StringBuilder textBuf = new StringBuilder(1000);
88
89 protected String userAgent = "gsa-crawler";
90
91 public void parse(final InputSource is) {
92 try {
93 final SAXParserFactory factory = SAXParserFactory.newInstance();
94 factory.setFeature(org.codelibs.fess.crawler.Constants.FEATURE_SECURE_PROCESSING, true);
95 factory.setFeature(org.codelibs.fess.crawler.Constants.FEATURE_EXTERNAL_GENERAL_ENTITIES, false);
96 factory.setFeature(org.codelibs.fess.crawler.Constants.FEATURE_EXTERNAL_PARAMETER_ENTITIES, false);
97 final SAXParser parser = factory.newSAXParser();
98 parser.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, StringUtil.EMPTY);
99 parser.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, StringUtil.EMPTY);
100 parser.parse(is, this);
101 } catch (final Exception e) {
102 throw new GsaConfigException("Failed to parse XML file.", e);
103 }
104 }
105
106 @Override
107 public void startDocument() throws SAXException {
108 tagQueue = new LinkedList<>();
109 labelList = new ArrayList<>();
110 labelType = null;
111 }
112
113 @Override
114 public void endDocument() throws SAXException {
115 globalParams.clear();
116 tagQueue.clear();
117 }
118
119 @Override
120 public void startElement(final String uri, final String localName, final String qName, final Attributes attributes)
121 throws SAXException {
122 if (logger.isDebugEnabled()) {
123 logger.debug("Start Element: {}", qName);
124 }
125 if (tagQueue.isEmpty() && !"eef".equalsIgnoreCase(qName)) {
126 throw new GsaConfigException("Invalid format.");
127 }
128 if (COLLECTION.equalsIgnoreCase(qName) && COLLECTIONS.equalsIgnoreCase(tagQueue.peekLast())) {
129 final long now = System.currentTimeMillis();
130 final String name = attributes.getValue("Name");
131 labelType = new LabelType();
132 labelType.setName(name);
133 labelType.setValue(name);
134 labelType.setPermissions(new String[] { "Rguest" });
135 labelType.setCreatedBy(Constants.SYSTEM_USER);
136 labelType.setCreatedTime(now);
137 labelType.setUpdatedBy(Constants.SYSTEM_USER);
138 labelType.setUpdatedTime(now);
139 }
140 tagQueue.offer(qName);
141 }
142
143 @Override
144 public void endElement(final String uri, final String localName, final String qName) throws SAXException {
145 if (logger.isDebugEnabled()) {
146 logger.debug("End Element: {}", qName);
147 }
148 if (GOOD_URLS.equalsIgnoreCase(qName)) {
149 if (labelType != null) {
150 labelType.setIncludedPaths(parseFilterPaths(textBuf.toString(), true, true));
151 } else if (GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
152 globalParams.put(GOOD_URLS, textBuf.toString());
153 }
154 } else if (BAD_URLS.equalsIgnoreCase(qName)) {
155 if (labelType != null) {
156 labelType.setExcludedPaths(parseFilterPaths(textBuf.toString(), true, true));
157 } else if (GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
158 globalParams.put(BAD_URLS, textBuf.toString());
159 }
160 } else if (START_URLS.equalsIgnoreCase(qName) && GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
161 globalParams.put(START_URLS, textBuf.toString());
162 } else if (labelType != null && COLLECTION.equalsIgnoreCase(qName)) {
163 labelList.add(labelType);
164 labelType = null;
165 } else if (GLOBALPARAMS.equalsIgnoreCase(qName)) {
166 final Object startUrls = globalParams.get(START_URLS);
167 if (startUrls != null) {
168 final long now = System.currentTimeMillis();
169 final List<String> urlList = split(startUrls.toString(), "\n")
170 .get(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).collect(Collectors.toList()));
171
172 final String webUrls = urlList.stream().filter(s -> Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p)))
173 .collect(Collectors.joining("\n"));
174 if (StringUtil.isNotBlank(webUrls)) {
175 webConfig = new WebConfig();
176 webConfig.setName("Default");
177 webConfig.setAvailable(true);
178 webConfig.setBoost(1.0f);
179 webConfig.setConfigParameter(StringUtil.EMPTY);
180 webConfig.setIntervalTime(1000);
181 webConfig.setNumOfThread(3);
182 webConfig.setSortOrder(1);
183 webConfig.setUrls(webUrls);
184 webConfig.setIncludedUrls(parseFilterPaths(globalParams.get(GOOD_URLS), true, false));
185 webConfig.setIncludedDocUrls(StringUtil.EMPTY);
186 webConfig.setExcludedUrls(parseFilterPaths(globalParams.get(BAD_URLS), true, false));
187 webConfig.setExcludedDocUrls(StringUtil.EMPTY);
188 webConfig.setUserAgent(userAgent);
189 webConfig.setPermissions(new String[] { "Rguest" });
190 webConfig.setCreatedBy(Constants.SYSTEM_USER);
191 webConfig.setCreatedTime(now);
192 webConfig.setUpdatedBy(Constants.SYSTEM_USER);
193 webConfig.setUpdatedTime(now);
194 }
195
196 final String fileUrls = urlList.stream().filter(s -> Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p)))
197 .collect(Collectors.joining("\n"));
198 if (StringUtil.isNotBlank(fileUrls)) {
199 fileConfig = new FileConfig();
200 fileConfig.setName("Default");
201 fileConfig.setAvailable(true);
202 fileConfig.setBoost(1.0f);
203 fileConfig.setConfigParameter(StringUtil.EMPTY);
204 fileConfig.setIntervalTime(0);
205 fileConfig.setNumOfThread(5);
206 fileConfig.setSortOrder(2);
207 fileConfig.setPaths(fileUrls);
208 fileConfig.setIncludedPaths(parseFilterPaths(globalParams.get(GOOD_URLS), false, true));
209 fileConfig.setIncludedDocPaths(StringUtil.EMPTY);
210 fileConfig.setExcludedPaths(parseFilterPaths(globalParams.get(BAD_URLS), false, true));
211 fileConfig.setExcludedDocPaths(StringUtil.EMPTY);
212 fileConfig.setPermissions(new String[] { "Rguest" });
213 fileConfig.setCreatedBy(Constants.SYSTEM_USER);
214 fileConfig.setCreatedTime(now);
215 fileConfig.setUpdatedBy(Constants.SYSTEM_USER);
216 fileConfig.setUpdatedTime(now);
217 }
218 }
219 } else if ("user_agent".equalsIgnoreCase(qName) && GLOBALPARAMS.equalsIgnoreCase(tagQueue.get(tagQueue.size() - 2))) {
220 userAgent = textBuf.toString().trim();
221 }
222 tagQueue.pollLast();
223 textBuf.setLength(0);
224 }
225
226 @Override
227 public void characters(final char[] ch, final int start, final int length) throws SAXException {
228 final String text = new String(ch, start, length);
229 if (logger.isDebugEnabled()) {
230 logger.debug("Text: {}", text);
231 }
232 textBuf.append(text);
233 }
234
235 protected String parseFilterPaths(final String text, final boolean web, final boolean file) {
236 return split(text, "\n")
237 .get(stream -> stream.map(String::trim).filter(StringUtil::isNotBlank).map(this::getFilterPath).filter(s -> {
238 if (StringUtil.isBlank(s)) {
239 return false;
240 }
241 if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))) {
242 return web;
243 }
244 if (Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) {
245 return file;
246 }
247 return true;
248 }).collect(Collectors.joining("\n")));
249 }
250
251 protected String getFilterPath(final String s) {
252 if (s.startsWith("#")) {
253 return StringUtil.EMPTY;
254 }
255 if (s.startsWith(CONTAINS)) {
256 final String v = s.substring(CONTAINS.length());
257 final StringBuilder buf = new StringBuilder(100);
258 return ".*" + appendFileterPath(buf, escape(v)) + ".*";
259 }
260 if (s.startsWith(REGEXP_IGNORE_CASE)) {
261 final String v = s.substring(REGEXP_IGNORE_CASE.length());
262 final StringBuilder buf = new StringBuilder(100);
263 buf.append("(?i)");
264 return appendFileterPath(buf, unescape(v));
265 } else if (s.startsWith(REGEXP_CASE)) {
266 final String v = s.substring(REGEXP_CASE.length());
267 final StringBuilder buf = new StringBuilder(100);
268 return appendFileterPath(buf, unescape(v));
269 } else if (s.startsWith(REGEXP)) {
270 final String v = s.substring(REGEXP.length());
271 final StringBuilder buf = new StringBuilder(100);
272 return appendFileterPath(buf, unescape(v));
273 } else if (Arrays.stream(webProtocols).anyMatch(p -> s.startsWith(p))
274 || Arrays.stream(fileProtocols).anyMatch(p -> s.startsWith(p))) {
275 return escape(s) + ".*";
276 } else {
277 final StringBuilder buf = new StringBuilder(100);
278 return appendFileterPath(buf, escape(s));
279 }
280 }
281
282 protected String escape(final String s) {
283 if (s.startsWith("#")) {
284 return StringUtil.EMPTY;
285 }
286 if (s.startsWith("^") && s.endsWith("$")) {
287 return "^" + Pattern.quote(s.substring(1, s.length() - 1)) + "$";
288 }
289 if (s.startsWith("^")) {
290 return "^" + Pattern.quote(s.substring(1));
291 } else if (s.endsWith("$")) {
292 return Pattern.quote(s.substring(0, s.length() - 1)) + "$";
293 }
294 return Pattern.quote(s);
295 }
296
297 protected String unescape(final String s) {
298 return s.replace("\\\\", "\\");
299 }
300
301 protected String appendFileterPath(final StringBuilder buf, final String v) {
302 if (StringUtil.isBlank(v)) {
303 return StringUtil.EMPTY;
304 }
305
306 if (v.startsWith("^")) {
307 buf.append(v);
308 if (!v.endsWith("$")) {
309 buf.append(".*");
310 }
311 } else if (v.endsWith("$")) {
312 buf.append(".*");
313 buf.append(v);
314 } else if (v.endsWith("/\\E")) {
315 buf.append(".*");
316 buf.append(v);
317 buf.append(".*");
318 } else {
319 buf.append(v);
320 }
321 return buf.toString();
322 }
323
324 public void setWebProtocols(final String[] webProtocols) {
325 this.webProtocols = webProtocols;
326 }
327
328 public void setFileProtocols(final String[] fileProtocols) {
329 this.fileProtocols = fileProtocols;
330 }
331
332 @Override
333 public String toString() {
334 return "GsaConfigParser [labelList=" + labelList + ", webConfig=" + webConfig + ", fileConfig=" + fileConfig + "]";
335 }
336
337 public OptionalEntity<WebConfig> getWebConfig() {
338 return OptionalUtil.ofNullable(webConfig);
339 }
340
341 public OptionalEntity<FileConfig> getFileConfig() {
342 return OptionalUtil.ofNullable(fileConfig);
343 }
344
345 public LabelType[] getLabelTypes() {
346 return labelList.toArray(new LabelType[labelList.size()]);
347 }
348
349 }