1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.api.gsa;
17
18 import java.io.IOException;
19 import java.io.UnsupportedEncodingException;
20 import java.net.URLDecoder;
21 import java.net.URLEncoder;
22 import java.text.SimpleDateFormat;
23 import java.util.ArrayList;
24 import java.util.Arrays;
25 import java.util.Collections;
26 import java.util.Date;
27 import java.util.Enumeration;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Locale;
31 import java.util.Map;
32 import java.util.Map.Entry;
33 import java.util.regex.Matcher;
34 import java.util.regex.Pattern;
35
36 import javax.annotation.PostConstruct;
37 import javax.servlet.FilterChain;
38 import javax.servlet.ServletException;
39 import javax.servlet.http.HttpServletRequest;
40 import javax.servlet.http.HttpServletResponse;
41
42 import org.apache.commons.text.StringEscapeUtils;
43 import org.apache.logging.log4j.LogManager;
44 import org.apache.logging.log4j.Logger;
45 import org.codelibs.core.CoreLibConstants;
46 import org.codelibs.core.lang.StringUtil;
47 import org.codelibs.fess.Constants;
48 import org.codelibs.fess.api.BaseApiManager;
49 import org.codelibs.fess.api.WebApiRequest;
50 import org.codelibs.fess.entity.FacetInfo;
51 import org.codelibs.fess.entity.GeoInfo;
52 import org.codelibs.fess.entity.HighlightInfo;
53 import org.codelibs.fess.entity.SearchRenderData;
54 import org.codelibs.fess.entity.SearchRequestParams;
55 import org.codelibs.fess.exception.InvalidAccessTokenException;
56 import org.codelibs.fess.helper.SearchHelper;
57 import org.codelibs.fess.mylasta.direction.FessConfig;
58 import org.codelibs.fess.util.ComponentUtil;
59 import org.codelibs.fess.util.DocumentUtil;
60 import org.dbflute.optional.OptionalThing;
61
62 public class GsaApiManager extends BaseApiManager {
63 private static final Logger logger = LogManager.getLogger(GsaApiManager.class);
64
65 private static final String OUTPUT_XML = "xml";
66
67
68 protected String gsaPathPrefix = "/gsa";
69
70 @PostConstruct
71 public void register() {
72 if (logger.isInfoEnabled()) {
73 logger.info("Load {}", this.getClass().getSimpleName());
74 }
75 ComponentUtil.getWebApiManagerFactory().add(this);
76 }
77
78 @Override
79 public boolean matches(final HttpServletRequest request) {
80 final FessConfig fessConfig = ComponentUtil.getFessConfig();
81 if (!fessConfig.isWebApiGsa()) {
82 return false;
83 }
84
85 final String servletPath = request.getServletPath();
86 return servletPath.startsWith(gsaPathPrefix);
87 }
88
89 @Override
90 public void process(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain)
91 throws IOException, ServletException {
92 switch (getFormatType(request)) {
93 case SEARCH:
94 processSearchRequest(request, response, chain);
95 break;
96 default:
97 writeXmlResponse(99, false, StringUtil.EMPTY, "Not found.");
98 break;
99 }
100 }
101
102 protected void appendParam(final StringBuilder buf, final String name, final String value) throws UnsupportedEncodingException {
103 appendParam(buf, name, value, URLEncoder.encode(value, Constants.UTF_8));
104 }
105
106 protected void appendParam(final StringBuilder buf, final String name, final String value, final String original) {
107 buf.append("<PARAM name=\"");
108 buf.append(escapeXml(name));
109 buf.append("\" value=\"");
110 buf.append(escapeXml(value));
111 buf.append("\" original_value=\"");
112 buf.append(escapeXml(original));
113 buf.append("\"/>");
114 }
115
116 protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
117 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
118 final FessConfig fessConfig = ComponentUtil.getFessConfig();
119 final boolean xmlDtd = OUTPUT_XML.equals(request.getParameter("output"));
120
121 if (!fessConfig.isAcceptedSearchReferer(request.getHeader("referer"))) {
122 writeXmlResponse(99, xmlDtd, StringUtil.EMPTY, "Referer is invalid.");
123 return;
124 }
125
126 int status = 0;
127 String errMsg = StringUtil.EMPTY;
128 String query = null;
129 final StringBuilder buf = new StringBuilder(1000);
130 request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_GSA);
131 try {
132 final SearchRenderData data = new SearchRenderData();
133 final GsaRequestParams params = new GsaRequestParams(request, fessConfig);
134 query = params.getQuery();
135 searchHelper.search(params, data, OptionalThing.empty());
136 final String execTime = data.getExecTime();
137 final long allRecordCount = data.getAllRecordCount();
138 final List<Map<String, Object>> documentItems = data.getDocumentItems();
139
140 final List<String> getFields = new ArrayList<>();
141
142 final String getFieldsParam = request.getParameter("getfields");
143 if (StringUtil.isNotBlank(getFieldsParam)) {
144 getFields.addAll(Arrays.asList(getFieldsParam.split("\\.")));
145 }
146 final StringBuilder requestUri = new StringBuilder(request.getRequestURI());
147 if (request.getQueryString() != null) {
148 requestUri.append("?").append(request.getQueryString());
149 }
150 final String uriQueryString = requestUri.toString();
151
152 final String ie = request.getCharacterEncoding();
153 final String oe = "UTF-8";
154
155 final String ip = ComponentUtil.getViewHelper().getClientIp(request);
156
157 buf.append("<TM>");
158 buf.append(execTime);
159 buf.append("</TM>");
160 buf.append("<Q>");
161 buf.append(escapeXml(query));
162 buf.append("</Q>");
163 for (final Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
164 final String[] values = entry.getValue();
165 if (values == null) {
166 continue;
167 }
168 final String key = entry.getKey();
169 if ("sort".equals(key)) {
170 continue;
171 }
172 for (final String value : values) {
173 appendParam(buf, key, value);
174 }
175 }
176 appendParam(buf, "ie", ie);
177 if (request.getParameter("oe") == null) {
178 appendParam(buf, "oe", oe);
179 }
180 final String[] langs = params.getLanguages();
181 if (langs.length > 0) {
182 appendParam(buf, "inlang", langs[0]);
183 appendParam(buf, "ulang", langs[0]);
184 }
185 appendParam(buf, "ip", ip);
186 appendParam(buf, "access", "p");
187 appendParam(buf, "sort", params.getSortParam(), params.getSortParam());
188 appendParam(buf, "entqr", "3");
189 appendParam(buf, "entqrm", "0");
190 appendParam(buf, "wc", "200");
191 appendParam(buf, "wc_mc", "1");
192 if (!documentItems.isEmpty()) {
193 buf.append("<RES SN=\"");
194 buf.append(data.getCurrentStartRecordNumber());
195 buf.append("\" EN=\"");
196 buf.append(data.getCurrentEndRecordNumber());
197 buf.append("\">");
198 buf.append("<M>");
199 buf.append(allRecordCount);
200 buf.append("</M>");
201 if (data.isExistPrevPage() || data.isExistNextPage()) {
202 buf.append("<NB>");
203 if (data.isExistPrevPage()) {
204 long s = data.getCurrentStartRecordNumber() - data.getPageSize() - 1;
205 if (s < 0) {
206 s = 0;
207 }
208 buf.append("<PU>");
209 buf.append(escapeXml(uriQueryString.replaceFirst("start=([0-9]+)", "start=" + s)));
210 buf.append("</PU>");
211 }
212 if (data.isExistNextPage()) {
213 buf.append("<NU>");
214 buf.append(escapeXml(uriQueryString.replaceFirst("start=([0-9]+)", "start=" + data.getCurrentEndRecordNumber())));
215 buf.append("</NU>");
216 }
217 buf.append("</NB>");
218 }
219 long recordNumber = data.getCurrentStartRecordNumber();
220 for (final Map<String, Object> document : documentItems) {
221 buf.append("<R N=\"");
222 buf.append(recordNumber);
223 buf.append("\">");
224 final String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
225 document.put("UE", url);
226 document.put("U", URLDecoder.decode(url, Constants.UTF_8));
227 document.put("T", DocumentUtil.getValue(document, fessConfig.getResponseFieldContentTitle(), String.class));
228 final float score = DocumentUtil.getValue(document, Constants.SCORE, Float.class, Float.NaN);
229 int rk = 10;
230 if (!Float.isNaN(score)) {
231 if (score < 0.0) {
232 rk = 0;
233 } else if (score > 1.0) {
234 rk = 10;
235 } else {
236 rk = (int) (score * 10.0);
237 }
238 }
239 document.put("RK", rk);
240 document.put("S", DocumentUtil.getValue(document, fessConfig.getResponseFieldContentDescription(), String.class,
241 StringUtil.EMPTY));
242 final String lang = DocumentUtil.getValue(document, fessConfig.getIndexFieldLang(), String.class);
243 if (StringUtil.isNotBlank(lang)) {
244 document.put("LANG", lang);
245 document.remove(fessConfig.getIndexFieldLang());
246 }
247 final String gsaMetaPrefix = fessConfig.getQueryGsaMetaPrefix();
248 for (final Map.Entry<String, Object> entry : document.entrySet()) {
249 final String name = entry.getKey();
250 if (StringUtil.isNotBlank(name) && entry.getValue() != null && fessConfig.isGsaResponseFields(name)) {
251 if (name.startsWith(gsaMetaPrefix)) {
252 final String tagName = name.replaceFirst("^" + gsaMetaPrefix, StringUtil.EMPTY);
253 if (getFields.contains(tagName)) {
254 buf.append("<MT N=\"");
255 buf.append(tagName);
256 buf.append("\" V=\"");
257 buf.append(escapeXml(entry.getValue()));
258 buf.append("\"/>");
259 }
260 } else {
261 final String tagName = name;
262 buf.append('<');
263 buf.append(tagName);
264 buf.append('>');
265 buf.append(escapeXml(entry.getValue()));
266 buf.append("</");
267 buf.append(tagName);
268 buf.append('>');
269 }
270 }
271 }
272 buf.append("<ENT_SOURCE>").append(escapeXml("FESS")).append("</ENT_SOURCE>");
273 String lastModified = DocumentUtil.getValue(document, fessConfig.getIndexFieldLastModified(), String.class);
274 if (StringUtil.isBlank(lastModified)) {
275 lastModified = StringUtil.EMPTY;
276 }
277 lastModified = lastModified.split("T")[0];
278 buf.append("<FS NAME=\"date\" VALUE=\"").append(escapeXml(lastModified)).append("\"/>");
279
280 buf.append("<HAS>");
281 buf.append("<L/>");
282 buf.append("<C SZ=\"");
283 buf.append(DocumentUtil.getValue(document, fessConfig.getIndexFieldContentLength(), Long.class, (long) 0).longValue()
284 / 1000);
285 document.remove(fessConfig.getIndexFieldContentLength());
286 buf.append("k\" CID=\"");
287 buf.append(DocumentUtil.getValue(document, fessConfig.getIndexFieldDocId(), String.class));
288 document.remove(fessConfig.getIndexFieldDocId());
289 buf.append("\" ENC=\"");
290 final String charsetField = fessConfig.getQueryGsaIndexFieldCharset();
291 String charset = DocumentUtil.getValue(document, charsetField, String.class);
292 document.remove(charsetField);
293 if (StringUtil.isBlank(charset)) {
294 final String contentTypeField = fessConfig.getQueryGsaIndexFieldContentType();
295 charset = DocumentUtil.getValue(document, contentTypeField, String.class);
296 document.remove(contentTypeField);
297 if (StringUtil.isNotBlank(charset)) {
298 final Matcher m = Pattern.compile(".*;\\s*charset=(.+)").matcher(charset);
299 if (m.matches()) {
300 charset = m.group(1);
301 }
302 }
303 if (StringUtil.isBlank(charset)) {
304 charset = Constants.UTF_8;
305 }
306 }
307 buf.append(charset);
308 buf.append("\"/>");
309 buf.append("</HAS>");
310 buf.append("</R>");
311 recordNumber++;
312 }
313 buf.append("</RES>");
314 }
315 } catch (final Exception e) {
316 status = 1;
317 errMsg = e.getMessage();
318 if (errMsg == null) {
319 errMsg = e.getClass().getName();
320 }
321 if (logger.isDebugEnabled()) {
322 logger.debug("Failed to process a search request.", e);
323 }
324 if (e instanceof InvalidAccessTokenException) {
325 final InvalidAccessTokenException iate = (InvalidAccessTokenException) e;
326 response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
327 response.setHeader("WWW-Authenticate", "Bearer error=\"" + iate.getType() + "\"");
328 }
329 }
330
331 writeXmlResponse(status, xmlDtd, buf.toString(), errMsg);
332 }
333
334 protected void writeXmlResponse(final int status, final boolean xmlDtd, final String body, final String errMsg) {
335 final StringBuilder buf = new StringBuilder(1000);
336 buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>");
337 if (xmlDtd) {
338 buf.append("<!DOCTYPE GSP SYSTEM \"google.dtd\">");
339 }
340 buf.append("<GSP VER=\"");
341 buf.append(Constants.GSA_API_VERSION);
342 buf.append("\">");
343 if (status == 0) {
344 buf.append(body);
345 } else {
346 buf.append("<message>");
347 buf.append(escapeXml(errMsg));
348 buf.append("</message>");
349 }
350 buf.append("</GSP>");
351 write(buf.toString(), "text/xml", Constants.UTF_8);
352
353 }
354
355 protected String escapeXml(final Object obj) {
356 final StringBuilder buf = new StringBuilder(255);
357 if (obj instanceof List<?>) {
358 buf.append("<list>");
359 for (final Object child : (List<?>) obj) {
360 buf.append("<item>").append(escapeXml(child)).append("</item>");
361 }
362 buf.append("</list>");
363 } else if (obj instanceof Map<?, ?>) {
364 buf.append("<data>");
365 for (final Map.Entry<?, ?> entry : ((Map<?, ?>) obj).entrySet()) {
366
367 buf.append("<name>").append(escapeXml(entry.getKey())).append("</name><value>").append(escapeXml(entry.getValue()))
368 .append("</value>");
369 }
370 buf.append("</data>");
371 } else if (obj instanceof Date) {
372 final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
373 buf.append(StringEscapeUtils.escapeXml10(sdf.format(obj)));
374 } else if (obj != null) {
375 buf.append(StringEscapeUtils.escapeXml10(obj.toString()));
376 }
377 return buf.toString();
378 }
379
380 public String getXmlPathPrefix() {
381 return gsaPathPrefix;
382 }
383
384 public void setXmlPathPrefix(final String xmlPathPrefix) {
385 this.gsaPathPrefix = xmlPathPrefix;
386 }
387
388 static class WrappedWebApiRequest extends WebApiRequest {
389 private Map<String, String[]> parameters;
390 private final Map<String, String[]> extraParameters;
391
392 public WrappedWebApiRequest(final HttpServletRequest request, final String servletPath, final Map<String, String[]> extraParams) {
393 super(request, servletPath);
394 extraParameters = extraParams;
395 }
396
397 @Override
398 public String getParameter(final String name) {
399 final String[] values = getParameterMap().get(name);
400 if (values != null) {
401 return values[0];
402 }
403 return super.getParameter(name);
404 }
405
406 @Override
407 public Map<String, String[]> getParameterMap() {
408 if (parameters == null) {
409 parameters = new HashMap<>();
410 parameters.putAll(super.getParameterMap());
411
412 parameters.putAll(extraParameters);
413 }
414 return Collections.unmodifiableMap(parameters);
415 }
416
417 @Override
418 public Enumeration<String> getParameterNames() {
419 return Collections.enumeration(getParameterMap().keySet());
420 }
421
422 @Override
423 public String[] getParameterValues(final String name) {
424 return getParameterMap().get(name);
425 }
426 }
427
428 protected static class GsaRequestParams extends SearchRequestParams {
429
430 private final HttpServletRequest request;
431
432 private final FessConfig fessConfig;
433
434 private int startPosition = -1;
435
436 private int pageSize = -1;
437
438 private String sortParam;
439
440 protected GsaRequestParams(final HttpServletRequest request, final FessConfig fessConfig) {
441 this.request = request;
442 this.fessConfig = fessConfig;
443 this.sortParam = request.getParameter("sort");
444 if (this.sortParam == null) {
445 this.sortParam = fessConfig.getQueryGsaDefaultSort();
446 }
447 }
448
449 @Override
450 public String getTrackTotalHits() {
451 return request.getParameter(Constants.TRACK_TOTAL_HITS);
452 }
453
454 @Override
455 public String getQuery() {
456 return request.getParameter("q");
457 }
458
459 @Override
460 public String[] getExtraQueries() {
461 final List<String> queryList = new ArrayList<>();
462 for (final String s : getParamValueArray(request, "ex_q")) {
463 queryList.add(s.trim());
464 }
465 final String gsaMetaPrefix = fessConfig.getQueryGsaMetaPrefix();
466 final String requiredFields = request.getParameter("requiredfields");
467 if (StringUtil.isNotBlank(requiredFields)) {
468 queryList.add(gsaMetaPrefix + requiredFields.replace(".", " AND " + gsaMetaPrefix).replace("|", " OR " + gsaMetaPrefix));
469 }
470 return queryList.toArray(new String[queryList.size()]);
471 }
472
473 @Override
474 public Map<String, String[]> getFields() {
475 final Map<String, String[]> fields = new HashMap<>();
476 for (final Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
477 final String key = entry.getKey();
478 if (key.startsWith("fields.")) {
479 final String[] value = simplifyArray(entry.getValue());
480 fields.put(key.substring("fields.".length()), value);
481 } else if ("site".equals(key)) {
482 final String[] value = simplifyArray(entry.getValue());
483 fields.put("label", value);
484 }
485 }
486 return fields;
487 }
488
489 @Override
490 public Map<String, String[]> getConditions() {
491 final Map<String, String[]> conditions = new HashMap<>();
492 for (final Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
493 final String key = entry.getKey();
494 if (key.startsWith("as_")) {
495 final String[] value = simplifyArray(entry.getValue());
496 conditions.put(key.substring("as_".length()), value);
497 }
498 }
499 return conditions;
500 }
501
502 @Override
503 public String[] getLanguages() {
504 final String[] langs = getParamValueArray(request, "ulang");
505 if (langs.length > 0) {
506 return langs;
507 }
508 if (request.getHeader("Accept-Language") != null) {
509 return Collections.list(request.getLocales()).stream().map(Locale::getLanguage).toArray(n -> new String[n]);
510 }
511 return new String[] { fessConfig.getQueryGsaDefaultLang() };
512 }
513
514 @Override
515 public GeoInfo getGeoInfo() {
516 return createGeoInfo(request);
517 }
518
519 @Override
520 public FacetInfo getFacetInfo() {
521 return createFacetInfo(request);
522 }
523
524 public String getSortParam() {
525 return sortParam;
526 }
527
528 @Override
529 public String getSort() {
530
531
532
533 if (StringUtil.isBlank(sortParam)) {
534 return null;
535 }
536
537 final String[] values = sortParam.split(":");
538 if (values.length > 0) {
539 if ("date".equals(values[0])) {
540 final StringBuilder buf = new StringBuilder();
541 buf.append(fessConfig.getIndexFieldTimestamp());
542 if (values.length > 1) {
543 if ("A".equals(values[1])) {
544 buf.append(".asc");
545 } else if ("D".equals(values[1])) {
546 buf.append(".desc");
547 }
548 }
549 buf.append(",score.desc");
550 return buf.toString();
551 }
552 if ("meta".equals(values[0]) && values.length > 1) {
553 final StringBuilder buf = new StringBuilder();
554 buf.append(fessConfig.getQueryGsaMetaPrefix() + values[1]);
555 if (values.length > 2) {
556 if ("A".equals(values[2])) {
557 buf.append(".asc");
558 } else if ("D".equals(values[2])) {
559 buf.append(".desc");
560 }
561 }
562 buf.append(",score.desc");
563 return buf.toString();
564 }
565 }
566
567 sortParam = "";
568 return null;
569 }
570
571 @Override
572 public int getStartPosition() {
573 if (startPosition != -1) {
574 return startPosition;
575 }
576
577 final String start = request.getParameter("start");
578 if (StringUtil.isBlank(start)) {
579 startPosition = fessConfig.getPagingSearchPageStartAsInteger();
580 } else {
581 try {
582 startPosition = Integer.parseInt(start);
583 } catch (final NumberFormatException e) {
584 startPosition = fessConfig.getPagingSearchPageStartAsInteger();
585 }
586 }
587 return startPosition;
588 }
589
590 @Override
591 public int getPageSize() {
592 if (pageSize != -1) {
593 return pageSize;
594 }
595
596 final String num = request.getParameter("num");
597 if (StringUtil.isBlank(num)) {
598 pageSize = fessConfig.getPagingSearchPageSizeAsInteger();
599 } else {
600 try {
601 pageSize = Integer.parseInt(num);
602 if (pageSize > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue() || pageSize <= 0) {
603 pageSize = fessConfig.getPagingSearchPageMaxSizeAsInteger();
604 }
605 } catch (final NumberFormatException e) {
606 pageSize = fessConfig.getPagingSearchPageSizeAsInteger();
607 }
608 }
609 return pageSize;
610 }
611
612 @Override
613 public Object getAttribute(final String name) {
614 return request.getAttribute(name);
615 }
616
617 @Override
618 public Locale getLocale() {
619 return Locale.ROOT;
620 }
621
622 @Override
623 public SearchRequestType getType() {
624 return SearchRequestType.GSA;
625 }
626
627 @Override
628 public String getSimilarDocHash() {
629 return request.getParameter("sdh");
630 }
631
632 @Override
633 public HighlightInfo getHighlightInfo() {
634 return ComponentUtil.getViewHelper().createHighlightInfo();
635 }
636 }
637
638 public void setGsaPathPrefix(final String gsaPathPrefix) {
639 this.gsaPathPrefix = gsaPathPrefix;
640 }
641
642 @Override
643 protected void writeHeaders(final HttpServletResponse response) {
644 ComponentUtil.getFessConfig().getApiGsaResponseHeaderList().forEach(e -> response.setHeader(e.getFirst(), e.getSecond()));
645 }
646 }