1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.api.gsa;
17
18 import java.io.IOException;
19 import java.net.URLDecoder;
20 import java.net.URLEncoder;
21 import java.text.SimpleDateFormat;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collections;
25 import java.util.Date;
26 import java.util.Enumeration;
27 import java.util.HashMap;
28 import java.util.List;
29 import java.util.Locale;
30 import java.util.Map;
31 import java.util.Map.Entry;
32 import java.util.regex.Matcher;
33 import java.util.regex.Pattern;
34
35 import javax.servlet.FilterChain;
36 import javax.servlet.ServletException;
37 import javax.servlet.http.HttpServletRequest;
38 import javax.servlet.http.HttpServletResponse;
39
40 import org.apache.commons.text.StringEscapeUtils;
41 import org.codelibs.core.CoreLibConstants;
42 import org.codelibs.core.lang.StringUtil;
43 import org.codelibs.fess.Constants;
44 import org.codelibs.fess.api.BaseApiManager;
45 import org.codelibs.fess.api.WebApiManager;
46 import org.codelibs.fess.api.WebApiRequest;
47 import org.codelibs.fess.app.service.SearchService;
48 import org.codelibs.fess.entity.FacetInfo;
49 import org.codelibs.fess.entity.GeoInfo;
50 import org.codelibs.fess.entity.SearchRenderData;
51 import org.codelibs.fess.entity.SearchRequestParams;
52 import org.codelibs.fess.exception.InvalidAccessTokenException;
53 import org.codelibs.fess.mylasta.direction.FessConfig;
54 import org.codelibs.fess.util.ComponentUtil;
55 import org.codelibs.fess.util.DocumentUtil;
56 import org.dbflute.optional.OptionalThing;
57 import org.slf4j.Logger;
58 import org.slf4j.LoggerFactory;
59
60 public class GsaApiManager extends BaseApiManager implements WebApiManager {
61 private static final Logger logger = LoggerFactory.getLogger(GsaApiManager.class);
62
63 private static final String GSA_META_SUFFIX = "_s";
64
65 protected String gsaPathPrefix = "/gsa";
66
67 protected String gsaMetaPrefix = "MT_";
68
69 protected String charsetField = "charset";
70
71 protected String contentTypeField = "content_type";
72
73 @Override
74 public boolean matches(final HttpServletRequest request) {
75 if (!ComponentUtil.getFessConfig().isWebApiGsa()) {
76 return false;
77 }
78 final String servletPath = request.getServletPath();
79 return servletPath.startsWith(gsaPathPrefix);
80 }
81
82 @Override
83 public void process(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) throws IOException,
84 ServletException {
85 final String formatType = request.getParameter("type");
86 switch (getFormatType(formatType)) {
87 case SEARCH:
88 processSearchRequest(request, response, chain);
89 break;
90 default:
91 writeXmlResponse(-1, false, StringUtil.EMPTY, "Not found.");
92 break;
93 }
94 }
95
96 protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
97 final SearchService searchService = ComponentUtil.getComponent(SearchService.class);
98 final FessConfig fessConfig = ComponentUtil.getFessConfig();
99
100 int status = 0;
101 String errMsg = StringUtil.EMPTY;
102 String query = null;
103 final StringBuilder buf = new StringBuilder(1000);
104 request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_XML);
105 boolean xmlDtd = false;
106 try {
107 final SearchRenderData data = new SearchRenderData();
108 final GsaRequestParams params = new GsaRequestParams(request, fessConfig);
109 query = params.getQuery();
110 searchService.search(params, data, OptionalThing.empty());
111 final String execTime = data.getExecTime();
112 final long allRecordCount = data.getAllRecordCount();
113 final List<Map<String, Object>> documentItems = data.getDocumentItems();
114
115 final List<String> getFields = new ArrayList<>();
116
117 final String getFieldsParam = request.getParameter("getfields");
118 if (StringUtil.isNotBlank(getFieldsParam)) {
119 getFields.addAll(Arrays.asList(getFieldsParam.split("\\.")));
120 }
121
122 if ("xml".equals(request.getParameter("output"))) {
123 xmlDtd = true;
124 }
125 final StringBuilder requestUri = new StringBuilder(request.getRequestURI());
126 if (request.getQueryString() != null) {
127 requestUri.append("?").append(request.getQueryString());
128 }
129 final String uriQueryString = requestUri.toString();
130
131 final String ie = request.getCharacterEncoding();
132 final String oe = "UTF-8";
133
134 final String ip = ComponentUtil.getViewHelper().getClientIp(request);
135 final String start = request.getParameter("start");
136 long startNumber = 1;
137 if (StringUtil.isNotBlank(start)) {
138 startNumber = Long.parseLong(start) + 1;
139 }
140 long endNumber = startNumber + data.getPageSize() - 1;
141 if (endNumber > allRecordCount) {
142 endNumber = allRecordCount;
143 }
144
145 buf.append("<Q>");
146 buf.append(escapeXml(query));
147 buf.append("</Q>");
148 buf.append("<TM>");
149 buf.append(execTime);
150 buf.append("</TM>");
151 for (final Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
152 final String[] values = entry.getValue();
153 if (values == null) {
154 continue;
155 }
156 final String key = entry.getKey();
157 for (final String value : values) {
158 buf.append("<PARAM name=\"");
159 buf.append(key);
160 buf.append("\" value=\"");
161 buf.append(value);
162 buf.append("\" original_value=\"");
163 buf.append(URLEncoder.encode(value, Constants.UTF_8));
164 buf.append("\"/>");
165 }
166 }
167 buf.append("<PARAM name=\"ie\" value=\"");
168 buf.append(ie);
169 buf.append("\" original_value=\"");
170 buf.append(URLEncoder.encode(ie, Constants.UTF_8));
171 buf.append("\"/>");
172 buf.append("<PARAM name=\"oe\" value=\"");
173 buf.append(oe);
174 buf.append("\" original_value=\"");
175 buf.append(URLEncoder.encode(ie, Constants.UTF_8));
176 buf.append("\"/>");
177 buf.append("<PARAM name=\"ip\" value=\"");
178 buf.append(ip);
179 buf.append("\" original_value=\"");
180 buf.append(URLEncoder.encode(ie, Constants.UTF_8));
181 buf.append("\"/>");
182 if (!documentItems.isEmpty()) {
183 buf.append("<RES SN=\"");
184 buf.append(startNumber);
185 buf.append("\" EN=\"");
186 buf.append(endNumber);
187 buf.append("\">");
188 buf.append("<M>");
189 buf.append(allRecordCount);
190 buf.append("</M>");
191 if (endNumber < allRecordCount) {
192 buf.append("<NB>");
193 buf.append("<NU>");
194 buf.append(escapeXml(uriQueryString.replaceFirst("start=([^&]+)", "start=" + endNumber)));
195 buf.append("</NU>");
196 buf.append("</NB>");
197 }
198 long recordNumber = startNumber;
199 for (final Map<String, Object> document : documentItems) {
200 buf.append("<R N=\"");
201 buf.append(recordNumber);
202 buf.append("\">");
203 final String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
204 document.remove(fessConfig.getIndexFieldUrl());
205 document.put("UE", url);
206 document.put("U", URLDecoder.decode(url, Constants.UTF_8));
207 document.put("T", DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class));
208 document.remove(fessConfig.getIndexFieldTitle());
209 final float score = DocumentUtil.getValue(document, fessConfig.getIndexFieldBoost(), Float.class, Float.valueOf(0));
210 document.remove(fessConfig.getIndexFieldBoost());
211 document.put("RK", (int) (score * 10));
212 document.put("S",
213 DocumentUtil
214 .getValue(document, fessConfig.getResponseFieldContentDescription(), String.class, StringUtil.EMPTY)
215 .replaceAll("<(/*)em>", "<$1b>"));
216 document.remove(fessConfig.getResponseFieldContentDescription());
217 final String lang = DocumentUtil.getValue(document, fessConfig.getIndexFieldLang(), String.class);
218 if (StringUtil.isNotBlank(lang)) {
219 document.put("LANG", lang);
220 document.remove(fessConfig.getIndexFieldLang());
221 }
222 for (final Map.Entry<String, Object> entry : document.entrySet()) {
223 final String name = entry.getKey();
224 if (StringUtil.isNotBlank(name) && entry.getValue() != null
225 && ComponentUtil.getQueryHelper().isApiResponseField(name)) {
226 if (name.startsWith(gsaMetaPrefix)) {
227 final String tagName = name.replaceAll("^" + gsaMetaPrefix, "").replaceAll(GSA_META_SUFFIX + "\\z", "");
228 if (getFields != null && getFields.contains(tagName)) {
229 buf.append("<MT N=\"");
230 buf.append(tagName);
231 buf.append("\" V=\"");
232 buf.append(escapeXml(entry.getValue()));
233 buf.append("\"/>");
234 }
235 } else {
236 final String tagName = name;
237 buf.append('<');
238 buf.append(tagName);
239 buf.append('>');
240 buf.append(escapeXml(entry.getValue()));
241 buf.append("</");
242 buf.append(tagName);
243 buf.append('>');
244 }
245 }
246 }
247 buf.append("<HAS>");
248 buf.append("<L/>");
249 buf.append("<C SZ=\"");
250 buf.append(DocumentUtil.getValue(document, fessConfig.getIndexFieldContentLength(), Long.class, Long.valueOf(0))
251 .longValue() / 1000);
252 document.remove(fessConfig.getIndexFieldContentLength());
253 buf.append("k\" CID=\"");
254 buf.append(DocumentUtil.getValue(document, fessConfig.getIndexFieldDocId(), String.class));
255 document.remove(fessConfig.getIndexFieldDocId());
256 buf.append("\" ENC=\"");
257 String charset = DocumentUtil.getValue(document, charsetField, String.class);
258 document.remove(charsetField);
259 if (StringUtil.isBlank(charset)) {
260 charset = DocumentUtil.getValue(document, contentTypeField, String.class);
261 document.remove(contentTypeField);
262 if (StringUtil.isNotBlank(charset)) {
263 final Matcher m = Pattern.compile(".*;\\s*charset=(.+)").matcher(charset);
264 if (m.matches()) {
265 charset = m.group(1);
266 }
267 }
268 if (StringUtil.isBlank(charset)) {
269 charset = Constants.UTF_8;
270 }
271 }
272 buf.append(charset);
273 buf.append("\"/>");
274 buf.append("</HAS>");
275 buf.append("</R>");
276 recordNumber++;
277 }
278 buf.append("</RES>");
279 }
280 } catch (final Exception e) {
281 status = 1;
282 errMsg = e.getMessage();
283 if (errMsg == null) {
284 errMsg = e.getClass().getName();
285 }
286 if (logger.isDebugEnabled()) {
287 logger.debug("Failed to process a search request.", e);
288 }
289 if (e instanceof InvalidAccessTokenException) {
290 final InvalidAccessTokenException iate = (InvalidAccessTokenException) e;
291 response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
292 response.setHeader("WWW-Authenticate", "Bearer error=\"" + iate.getType() + "\"");
293 }
294 }
295
296 writeXmlResponse(status, xmlDtd, buf.toString(), errMsg);
297 }
298
299 protected void writeXmlResponse(final int status, final boolean xmlDtd, final String body, final String errMsg) {
300 final StringBuilder buf = new StringBuilder(1000);
301 buf.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>");
302 if (xmlDtd) {
303 buf.append("<!DOCTYPE GSP SYSTEM \"google.dtd\">");
304 }
305 buf.append("<GSP VER=\"");
306 buf.append(Constants.GSA_API_VERSION);
307 buf.append("\">");
308 if (status == 0) {
309 buf.append(body);
310 } else {
311 buf.append("<message>");
312 buf.append(escapeXml(errMsg));
313 buf.append("</message>");
314 }
315 buf.append("</GSP>");
316 write(buf.toString(), "text/xml", Constants.UTF_8);
317
318 }
319
320 protected String escapeXml(final Object obj) {
321 final StringBuilder buf = new StringBuilder(255);
322 if (obj instanceof List<?>) {
323 buf.append("<list>");
324 for (final Object child : (List<?>) obj) {
325 buf.append("<item>").append(escapeXml(child)).append("</item>");
326 }
327 buf.append("</list>");
328 } else if (obj instanceof Map<?, ?>) {
329 buf.append("<data>");
330 for (final Map.Entry<?, ?> entry : ((Map<?, ?>) obj).entrySet()) {
331
332 buf.append("<name>").append(escapeXml(entry.getKey())).append("</name><value>").append(escapeXml(entry.getValue()))
333 .append("</value>");
334 }
335 buf.append("</data>");
336 } else if (obj instanceof Date) {
337 final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
338 buf.append(StringEscapeUtils.escapeXml10(sdf.format(obj)));
339 } else if (obj != null) {
340 buf.append(StringEscapeUtils.escapeXml10(obj.toString()));
341 }
342 return buf.toString();
343 }
344
345 public String getXmlPathPrefix() {
346 return gsaPathPrefix;
347 }
348
349 public void setXmlPathPrefix(final String xmlPathPrefix) {
350 this.gsaPathPrefix = xmlPathPrefix;
351 }
352
353 static class WrappedWebApiRequest extends WebApiRequest {
354 private Map<String, String[]> parameters;
355 private final Map<String, String[]> extraParameters;
356
357 public WrappedWebApiRequest(final HttpServletRequest request, final String servletPath, final Map<String, String[]> extraParams) {
358 super(request, servletPath);
359 extraParameters = extraParams;
360 }
361
362 @Override
363 public String getParameter(final String name) {
364 final String[] values = getParameterMap().get(name);
365 if (values != null) {
366 return values[0];
367 }
368 return super.getParameter(name);
369 }
370
371 @Override
372 public Map<String, String[]> getParameterMap() {
373 if (parameters == null) {
374 parameters = new HashMap<>();
375 parameters.putAll(super.getParameterMap());
376
377 parameters.putAll(extraParameters);
378 }
379 return Collections.unmodifiableMap(parameters);
380 }
381
382 @Override
383 public Enumeration<String> getParameterNames() {
384 return Collections.enumeration(getParameterMap().keySet());
385 }
386
387 @Override
388 public String[] getParameterValues(final String name) {
389 return getParameterMap().get(name);
390 }
391 }
392
393 protected static class GsaRequestParams implements SearchRequestParams {
394
395 private final HttpServletRequest request;
396
397 private final FessConfig fessConfig;
398
399 private int startPosition = -1;
400
401 private int pageSize = -1;
402
403 protected GsaRequestParams(final HttpServletRequest request, final FessConfig fessConfig) {
404 this.request = request;
405 this.fessConfig = fessConfig;
406 }
407
408 @Override
409 public String getQuery() {
410 return request.getParameter("q");
411 }
412
413 @Override
414 public String[] getExtraQueries() {
415 return getParamValueArray(request, "ex_q");
416 }
417
418 @Override
419 public Map<String, String[]> getFields() {
420 final Map<String, String[]> fields = new HashMap<>();
421 for (final Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
422 final String key = entry.getKey();
423 if (key.startsWith("fields.")) {
424 final String[] value = simplifyArray(entry.getValue());
425 fields.put(key.substring("fields.".length()), value);
426 }
427 }
428 return fields;
429 }
430
431 @Override
432 public String[] getLanguages() {
433 return getParamValueArray(request, "lang");
434 }
435
436 @Override
437 public GeoInfo getGeoInfo() {
438 return createGeoInfo(request);
439 }
440
441 @Override
442 public FacetInfo getFacetInfo() {
443 return createFacetInfo(request);
444 }
445
446 @Override
447 public String getSort() {
448 return request.getParameter("sort");
449 }
450
451 @Override
452 public int getStartPosition() {
453 if (startPosition != -1) {
454 return startPosition;
455 }
456
457 final String start = request.getParameter("start");
458 if (StringUtil.isBlank(start)) {
459 startPosition = fessConfig.getPagingSearchPageStartAsInteger();
460 } else {
461 try {
462 startPosition = Integer.parseInt(start);
463 } catch (final NumberFormatException e) {
464 startPosition = fessConfig.getPagingSearchPageStartAsInteger();
465 }
466 }
467 return startPosition;
468 }
469
470 @Override
471 public int getPageSize() {
472 if (pageSize != -1) {
473 return pageSize;
474 }
475
476 final String num = request.getParameter("num");
477 if (StringUtil.isBlank(num)) {
478 pageSize = fessConfig.getPagingSearchPageSizeAsInteger();
479 } else {
480 try {
481 pageSize = Integer.parseInt(num);
482 if (pageSize > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue() || pageSize <= 0) {
483 pageSize = fessConfig.getPagingSearchPageMaxSizeAsInteger();
484 }
485 } catch (final NumberFormatException e) {
486 pageSize = fessConfig.getPagingSearchPageSizeAsInteger();
487 }
488 }
489 return pageSize;
490 }
491
492 @Override
493 public Object getAttribute(final String name) {
494 return request.getAttribute(name);
495 }
496
497 @Override
498 public Locale getLocale() {
499 return Locale.ROOT;
500 }
501
502 @Override
503 public SearchRequestType getType() {
504 return SearchRequestType.GSA;
505 }
506
507 @Override
508 public String getSimilarDocHash() {
509 return request.getParameter("sdh");
510 }
511
512 }
513
514 public void setGsaPathPrefix(final String gsaPathPrefix) {
515 this.gsaPathPrefix = gsaPathPrefix;
516 }
517
518 public void setGsaMetaPrefix(final String gsaMetaPrefix) {
519 this.gsaMetaPrefix = gsaMetaPrefix;
520 }
521
522 public void setCharsetField(String charsetField) {
523 this.charsetField = charsetField;
524 }
525
526 public void setContentTypeField(String contentTypeField) {
527 this.contentTypeField = contentTypeField;
528 }
529 }