1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.api.json;
17
18 import static org.codelibs.core.stream.StreamUtil.stream;
19
20 import java.io.IOException;
21 import java.io.PrintWriter;
22 import java.io.StringWriter;
23 import java.net.URLDecoder;
24 import java.text.SimpleDateFormat;
25 import java.util.ArrayList;
26 import java.util.Collections;
27 import java.util.Date;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Locale;
31 import java.util.Map;
32 import java.util.UUID;
33 import java.util.function.Supplier;
34
35 import org.apache.commons.lang3.ArrayUtils;
36 import org.apache.commons.lang3.StringUtils;
37 import org.apache.commons.text.StringEscapeUtils;
38 import org.apache.logging.log4j.LogManager;
39 import org.apache.logging.log4j.Logger;
40 import org.codelibs.core.CoreLibConstants;
41 import org.codelibs.core.exception.IORuntimeException;
42 import org.codelibs.core.lang.StringUtil;
43 import org.codelibs.fess.Constants;
44 import org.codelibs.fess.api.BaseApiManager;
45 import org.codelibs.fess.app.service.FavoriteLogService;
46 import org.codelibs.fess.entity.FacetInfo;
47 import org.codelibs.fess.entity.GeoInfo;
48 import org.codelibs.fess.entity.HighlightInfo;
49 import org.codelibs.fess.entity.PingResponse;
50 import org.codelibs.fess.entity.SearchRenderData;
51 import org.codelibs.fess.entity.SearchRequestParams;
52 import org.codelibs.fess.entity.SearchRequestParams.SearchRequestType;
53 import org.codelibs.fess.exception.InvalidAccessTokenException;
54 import org.codelibs.fess.exception.InvalidQueryException;
55 import org.codelibs.fess.exception.ResultOffsetExceededException;
56 import org.codelibs.fess.exception.WebApiException;
57 import org.codelibs.fess.helper.LabelTypeHelper;
58 import org.codelibs.fess.helper.PopularWordHelper;
59 import org.codelibs.fess.helper.RelatedContentHelper;
60 import org.codelibs.fess.helper.RelatedQueryHelper;
61 import org.codelibs.fess.helper.RoleQueryHelper;
62 import org.codelibs.fess.helper.SearchHelper;
63 import org.codelibs.fess.helper.SuggestHelper;
64 import org.codelibs.fess.helper.SystemHelper;
65 import org.codelibs.fess.helper.UserInfoHelper;
66 import org.codelibs.fess.mylasta.direction.FessConfig;
67 import org.codelibs.fess.opensearch.client.SearchEngineClient;
68 import org.codelibs.fess.suggest.entity.SuggestItem;
69 import org.codelibs.fess.suggest.request.suggest.SuggestRequestBuilder;
70 import org.codelibs.fess.suggest.request.suggest.SuggestResponse;
71 import org.codelibs.fess.util.ComponentUtil;
72 import org.codelibs.fess.util.DocumentUtil;
73 import org.codelibs.fess.util.FacetResponse;
74 import org.codelibs.fess.util.FacetResponse.Field;
75 import org.dbflute.optional.OptionalThing;
76 import org.lastaflute.web.util.LaRequestUtil;
77 import org.lastaflute.web.util.LaResponseUtil;
78 import org.opensearch.script.Script;
79
80 import jakarta.annotation.PostConstruct;
81 import jakarta.servlet.FilterChain;
82 import jakarta.servlet.ServletException;
83 import jakarta.servlet.http.HttpServletRequest;
84 import jakarta.servlet.http.HttpServletResponse;
85
86
87
88
89 public class SearchApiManager extends BaseApiManager {
90
91 private static final Logger logger = LogManager.getLogger(SearchApiManager.class);
92
93
94
95
96 protected static final String MESSAGE_FIELD = "message";
97
98
99
100
101 protected static final String RESULT_FIELD = "result";
102
103 private static final String DOC_ID_FIELD = "doc_id";
104
105
106
107
108 protected static final String GET = "GET";
109
110
111
112
113 protected static final String POST = "POST";
114
115
116
117
118 protected String mimeType = "application/json";
119
120
121
122
123 public SearchApiManager() {
124 setPathPrefix("/api/v1");
125 }
126
127
128
129
130 @PostConstruct
131 public void register() {
132 if (logger.isInfoEnabled()) {
133 logger.info("Loaded {}", this.getClass().getSimpleName());
134 }
135 ComponentUtil.getWebApiManagerFactory().add(this);
136 }
137
138 @Override
139 protected FormatType detectFormatType(final HttpServletRequest request) {
140 final String servletPath = request.getServletPath();
141 final String[] values = servletPath.replaceAll("/+", "/").split("/");
142 final String value = values.length > 3 ? values[3] : null;
143 if (value == null) {
144 return FormatType.SEARCH;
145 }
146 final String type = value.toLowerCase(Locale.ROOT);
147 if ("documents".equals(type)) {
148 if (values.length > 5 && "favorite".equals(values[5])) {
149 request.setAttribute(DOC_ID_FIELD, values[4]);
150 return FormatType.FAVORITE;
151 }
152 if (values.length > 4 && "all".equals(values[4])) {
153 return FormatType.SCROLL;
154 }
155 return FormatType.SEARCH;
156 }
157 if ("labels".equals(type)) {
158 return FormatType.LABEL;
159 }
160 if ("popular-words".equals(type)) {
161 return FormatType.POPULARWORD;
162 }
163 if ("favorites".equals(type)) {
164 return FormatType.FAVORITES;
165 }
166 if ("health".equals(type)) {
167 return FormatType.PING;
168 }
169 if ("suggest-words".equals(type)) {
170 return FormatType.SUGGEST;
171 }
172
173 return FormatType.OTHER;
174 }
175
176 @Override
177 public boolean matches(final HttpServletRequest request) {
178 final FessConfig fessConfig = ComponentUtil.getFessConfig();
179 if (!fessConfig.isWebApiJson()) {
180 return false;
181 }
182
183 final String servletPath = request.getServletPath();
184 return servletPath.startsWith(pathPrefix);
185 }
186
187 @Override
188 public void process(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain)
189 throws IOException, ServletException {
190 switch (getFormatType(request)) {
191 case SEARCH:
192 processSearchRequest(request, response, chain);
193 break;
194 case LABEL:
195 processLabelRequest(request, response, chain);
196 break;
197 case POPULARWORD:
198 processPopularWordRequest(request, response, chain);
199 break;
200 case FAVORITE:
201 processFavoriteRequest(request, response, chain);
202 break;
203 case FAVORITES:
204 processFavoritesRequest(request, response, chain);
205 break;
206 case PING:
207 processPingRequest(request, response, chain);
208 break;
209 case SCROLL:
210 processScrollSearchRequest(request, response, chain);
211 break;
212 case SUGGEST:
213 processSuggestRequest(request, response, chain);
214 break;
215 default:
216 writeJsonResponse(HttpServletResponse.SC_NOT_FOUND, escapeJsonKeyValue(MESSAGE_FIELD, "Not found."));
217 break;
218 }
219 }
220
221
222
223
224
225
226
227 protected boolean acceptHttpMethod(final HttpServletRequest request, final String... methods) {
228 final String method = request.getMethod();
229 for (final String m : methods) {
230 if (m.equals(method)) {
231 return true;
232 }
233 }
234 writeJsonResponse(HttpServletResponse.SC_METHOD_NOT_ALLOWED, escapeJsonKeyValue(MESSAGE_FIELD, method + " is not allowed."));
235 return false;
236 }
237
238
239
240
241
242
243
244 protected void processScrollSearchRequest(final HttpServletRequest request, final HttpServletResponse response,
245 final FilterChain chain) {
246 if (!acceptHttpMethod(request, GET)) {
247 return;
248 }
249 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
250 final FessConfig fessConfig = ComponentUtil.getFessConfig();
251
252 if (!fessConfig.isAcceptedSearchReferer(request.getHeader("referer"))) {
253 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, escapeJsonKeyValue(MESSAGE_FIELD, "Referer is invalid."));
254 return;
255 }
256
257 if (!fessConfig.isApiSearchScroll()) {
258 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, escapeJsonKeyValue(MESSAGE_FIELD, "Scroll Search is not available."));
259 return;
260 }
261
262 final StringBuilder buf = new StringBuilder(1000);
263 request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
264 final JsonRequestParams params = new JsonRequestParams(request, fessConfig);
265 try {
266 response.setContentType("application/x-ndjson; charset=UTF-8");
267 final long count = searchHelper.scrollSearch(params, doc -> {
268 buf.setLength(0);
269 buf.append('{');
270 boolean first2 = true;
271 for (final Map.Entry<String, Object> entry : doc.entrySet()) {
272 final String name = entry.getKey();
273 if (StringUtil.isNotBlank(name) && entry.getValue() != null) {
274 if (!first2) {
275 buf.append(',');
276 } else {
277 first2 = false;
278 }
279 buf.append(escapeJson(name));
280 buf.append(':');
281 buf.append(escapeJson(entry.getValue()));
282 }
283 }
284 buf.append('}');
285 buf.append('\n');
286 try {
287 response.getWriter().print(buf.toString());
288 } catch (final IOException e) {
289 throw new IORuntimeException(e);
290 }
291 return true;
292 }, OptionalThing.empty());
293 response.flushBuffer();
294 if (logger.isDebugEnabled()) {
295 logger.debug("Loaded {} documents", count);
296 }
297 } catch (final InvalidQueryException | ResultOffsetExceededException e) {
298 if (logger.isDebugEnabled()) {
299 logger.debug("Failed to process a scroll request.", e);
300 }
301 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, e);
302 } catch (final Exception e) {
303 if (logger.isDebugEnabled()) {
304 logger.debug("Failed to process a scroll request.", e);
305 }
306 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
307 }
308
309 }
310
311
312
313
314
315
316
317 protected void processPingRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
318 if (!acceptHttpMethod(request, GET)) {
319 return;
320 }
321
322 final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
323 try {
324 final PingResponse pingResponse = searchEngineClient.ping();
325 writeJsonResponse(pingResponse.getStatus() == 0 ? HttpServletResponse.SC_OK : HttpServletResponse.SC_SERVICE_UNAVAILABLE,
326 "\"data\":" + pingResponse.getMessage());
327 } catch (final Exception e) {
328 if (logger.isDebugEnabled()) {
329 logger.debug("Failed to process a ping request.", e);
330 }
331 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
332 }
333 }
334
335
336
337
338
339
340
341 protected void processSearchRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
342 if (!acceptHttpMethod(request, GET)) {
343 return;
344 }
345
346 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
347 final FessConfig fessConfig = ComponentUtil.getFessConfig();
348 final RelatedQueryHelper relatedQueryHelper = ComponentUtil.getRelatedQueryHelper();
349 final RelatedContentHelper relatedContentHelper = ComponentUtil.getRelatedContentHelper();
350
351 String query = null;
352 final StringBuilder buf = new StringBuilder(1000);
353 request.setAttribute(Constants.SEARCH_LOG_ACCESS_TYPE, Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
354 try {
355 final SearchRenderData data = new SearchRenderData();
356 final JsonRequestParams params = new JsonRequestParams(request, fessConfig);
357 query = params.getQuery();
358 searchHelper.search(params, data, OptionalThing.empty());
359 final String execTime = data.getExecTime();
360 final String queryTime = Long.toString(data.getQueryTime());
361 final String pageSize = Integer.toString(data.getPageSize());
362 final String currentPageNumber = Integer.toString(data.getCurrentPageNumber());
363 final String allRecordCount = Long.toString(data.getAllRecordCount());
364 final String allRecordCountRelation = data.getAllRecordCountRelation();
365 final String allPageCount = Integer.toString(data.getAllPageCount());
366 final List<Map<String, Object>> documentItems = data.getDocumentItems();
367 final FacetResponse facetResponse = data.getFacetResponse();
368 final String queryId = data.getQueryId();
369 final String highlightParams = data.getAppendHighlightParams();
370 final boolean nextPage = data.isExistNextPage();
371 final boolean prevPage = data.isExistPrevPage();
372 final long startRecordNumber = data.getCurrentStartRecordNumber();
373 final long endRecordNumber = data.getCurrentEndRecordNumber();
374 final List<String> pageNumbers = data.getPageNumberList();
375 final boolean partial = data.isPartialResults();
376 final String searchQuery = data.getSearchQuery();
377 final long requestedTime = data.getRequestedTime();
378
379 buf.append("\"q\":");
380 buf.append(escapeJson(query));
381 buf.append(",\"query_id\":");
382 buf.append(escapeJson(queryId));
383 buf.append(",\"exec_time\":");
384 buf.append(execTime);
385 buf.append(",\"query_time\":");
386 buf.append(queryTime);
387 buf.append(',');
388 buf.append("\"page_size\":");
389 buf.append(pageSize);
390 buf.append(',');
391 buf.append("\"page_number\":");
392 buf.append(currentPageNumber);
393 buf.append(',');
394 buf.append("\"record_count\":");
395 buf.append(allRecordCount);
396 buf.append(',');
397 buf.append("\"record_count_relation\":");
398 buf.append(escapeJson(allRecordCountRelation));
399 buf.append(',');
400 buf.append("\"page_count\":");
401 buf.append(allPageCount);
402 buf.append(",\"highlight_params\":");
403 buf.append(escapeJson(highlightParams));
404 buf.append(",\"next_page\":");
405 buf.append(escapeJson(nextPage));
406 buf.append(",\"prev_page\":");
407 buf.append(escapeJson(prevPage));
408 buf.append(",\"start_record_number\":");
409 buf.append(startRecordNumber);
410 buf.append(",\"end_record_number\":");
411 buf.append(escapeJson(endRecordNumber));
412 buf.append(",\"page_numbers\":");
413 buf.append(escapeJson(pageNumbers));
414 buf.append(",\"partial\":");
415 buf.append(escapeJson(partial));
416 buf.append(",\"search_query\":");
417 buf.append(escapeJson(searchQuery));
418 buf.append(",\"requested_time\":");
419 buf.append(requestedTime);
420 final String[] relatedQueries = relatedQueryHelper.getRelatedQueries(params.getQuery());
421 buf.append(",\"related_query\":");
422 buf.append(escapeJson(relatedQueries));
423 final String[] relatedContents = relatedContentHelper.getRelatedContents(params.getQuery());
424 buf.append(",\"related_contents\":");
425 buf.append(escapeJson(relatedContents));
426 buf.append(',');
427 buf.append("\"data\":[");
428 if (!documentItems.isEmpty()) {
429 boolean first1 = true;
430 for (final Map<String, Object> document : documentItems) {
431 if (!first1) {
432 buf.append(',');
433 } else {
434 first1 = false;
435 }
436 buf.append('{');
437 boolean first2 = true;
438 for (final Map.Entry<String, Object> entry : document.entrySet()) {
439 final String name = entry.getKey();
440 if (StringUtil.isNotBlank(name) && entry.getValue() != null
441 && ComponentUtil.getQueryFieldConfig().isApiResponseField(name)) {
442 if (!first2) {
443 buf.append(',');
444 } else {
445 first2 = false;
446 }
447 buf.append(escapeJson(name));
448 buf.append(':');
449 buf.append(escapeJson(entry.getValue()));
450 }
451 }
452 buf.append('}');
453 }
454 }
455 buf.append(']');
456 if (facetResponse != null && facetResponse.hasFacetResponse()) {
457
458 buf.append(',');
459 buf.append("\"facet_field\":[");
460 if (facetResponse.getFieldList() != null) {
461 boolean first1 = true;
462 for (final Field field : facetResponse.getFieldList()) {
463 if (!first1) {
464 buf.append(',');
465 } else {
466 first1 = false;
467 }
468 buf.append("{\"name\":");
469 buf.append(escapeJson(field.getName()));
470 buf.append(",\"result\":[");
471 boolean first2 = true;
472 for (final Map.Entry<String, Long> entry : field.getValueCountMap().entrySet()) {
473 if (!first2) {
474 buf.append(',');
475 } else {
476 first2 = false;
477 }
478 buf.append("{\"value\":");
479 buf.append(escapeJson(entry.getKey()));
480 buf.append(",\"count\":");
481 buf.append(entry.getValue());
482 buf.append('}');
483 }
484 buf.append(']');
485 buf.append('}');
486 }
487 }
488 buf.append(']');
489
490 buf.append(',');
491 buf.append("\"facet_query\":[");
492 if (facetResponse.getQueryCountMap() != null) {
493 boolean first1 = true;
494 for (final Map.Entry<String, Long> entry : facetResponse.getQueryCountMap().entrySet()) {
495 if (!first1) {
496 buf.append(',');
497 } else {
498 first1 = false;
499 }
500 buf.append("{\"value\":");
501 buf.append(escapeJson(entry.getKey()));
502 buf.append(",\"count\":");
503 buf.append(entry.getValue());
504 buf.append('}');
505 }
506 }
507 buf.append(']');
508 }
509 writeJsonResponse(HttpServletResponse.SC_OK, buf.toString());
510 } catch (final InvalidQueryException | ResultOffsetExceededException e) {
511 if (logger.isDebugEnabled()) {
512 logger.debug("Failed to process a search request.", e);
513 }
514 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, e);
515 } catch (final Exception e) {
516 if (logger.isDebugEnabled()) {
517 logger.debug("Failed to process a search request.", e);
518 }
519 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
520 }
521 }
522
523
524
525
526
527
528 protected String detailedMessage(final Throwable t) {
529 if (t == null) {
530 return "Unknown";
531 }
532 Throwable target = t;
533 if (target.getCause() == null) {
534 return target.getClass().getSimpleName() + "[" + target.getMessage() + "]";
535 }
536 final StringBuilder sb = new StringBuilder();
537 while (target != null) {
538 sb.append(target.getClass().getSimpleName());
539 if (target.getMessage() != null) {
540 sb.append("[");
541 sb.append(target.getMessage());
542 sb.append("]");
543 }
544 sb.append("; ");
545 target = target.getCause();
546 if (target != null) {
547 sb.append("nested: ");
548 }
549 }
550 return sb.toString();
551 }
552
553
554
555
556
557
558
559 protected void processLabelRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
560 if (!acceptHttpMethod(request, GET)) {
561 return;
562 }
563
564 final LabelTypeHelper labelTypeHelper = ComponentUtil.getLabelTypeHelper();
565
566 final StringBuilder buf = new StringBuilder(255);
567 try {
568 final List<Map<String, String>> labelTypeItems = labelTypeHelper.getLabelTypeItemList(SearchRequestType.JSON,
569 request.getLocale() == null ? Locale.ROOT : request.getLocale());
570 buf.append("\"record_count\":");
571 buf.append(labelTypeItems.size());
572 if (!labelTypeItems.isEmpty()) {
573 buf.append(',');
574 buf.append("\"data\":[");
575 boolean first1 = true;
576 for (final Map<String, String> labelMap : labelTypeItems) {
577 if (!first1) {
578 buf.append(',');
579 } else {
580 first1 = false;
581 }
582 buf.append("{\"label\":");
583 buf.append(escapeJson(labelMap.get(Constants.ITEM_LABEL)));
584 buf.append(", \"value\":");
585 buf.append(escapeJson(labelMap.get(Constants.ITEM_VALUE)));
586 buf.append('}');
587 }
588 buf.append(']');
589 }
590
591 writeJsonResponse(HttpServletResponse.SC_OK, buf.toString());
592 } catch (final Exception e) {
593 if (logger.isDebugEnabled()) {
594 logger.debug("Failed to process a label request.", e);
595 }
596 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
597 }
598 }
599
600
601
602
603
604
605
606 protected void processPopularWordRequest(final HttpServletRequest request, final HttpServletResponse response,
607 final FilterChain chain) {
608 if (!acceptHttpMethod(request, GET)) {
609 return;
610 }
611
612 if (!ComponentUtil.getFessConfig().isWebApiPopularWord()) {
613 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, escapeJsonKeyValue(MESSAGE_FIELD, "Unsupported operation."));
614 return;
615 }
616
617 final String seed = request.getParameter("seed");
618 String[] tags = SearchRequestParams.getParamValueArray(request, "label");
619 final String key = ComponentUtil.getVirtualHostHelper().getVirtualHostKey();
620 if (StringUtil.isNotBlank(key)) {
621 tags = ArrayUtils.addAll(tags, key);
622 }
623 final String[] fields = request.getParameterValues("field");
624
625 final PopularWordHelper popularWordHelper = ComponentUtil.getPopularWordHelper();
626
627 final StringBuilder buf = new StringBuilder(255);
628 try {
629 final List<String> popularWordList =
630 popularWordHelper.getWordList(SearchRequestType.JSON, seed, tags, null, fields, StringUtil.EMPTY_STRINGS);
631 buf.append("\"record_count\":").append(popularWordList.size()).append(',');
632 buf.append("\"data\":[");
633 boolean first1 = true;
634 for (final String word : popularWordList) {
635 if (!first1) {
636 buf.append(',');
637 } else {
638 first1 = false;
639 }
640 buf.append(escapeJson(word));
641 }
642 buf.append(']');
643 writeJsonResponse(HttpServletResponse.SC_OK, buf.toString());
644 } catch (final WebApiException e) {
645 if (logger.isDebugEnabled()) {
646 logger.debug("Failed to process a popularWord request.", e);
647 }
648 writeJsonResponse(e.getStatusCode(), e);
649 } catch (final Exception e) {
650 if (logger.isDebugEnabled()) {
651 logger.debug("Failed to process a popularWord request.", e);
652 }
653 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
654 }
655 }
656
657
658
659
660
661
662
663 protected void processFavoriteRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
664 if (!acceptHttpMethod(request, POST)) {
665 return;
666 }
667
668 if (!ComponentUtil.getFessConfig().isUserFavorite()) {
669 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, escapeJsonKeyValue(MESSAGE_FIELD, "Unsupported operation."));
670 return;
671 }
672
673 final FessConfig fessConfig = ComponentUtil.getFessConfig();
674 final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
675 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
676 final FavoriteLogService favoriteLogService = ComponentUtil.getComponent(FavoriteLogService.class);
677 final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
678
679 try {
680 final Object docIdObj = request.getAttribute(DOC_ID_FIELD);
681 if (docIdObj == null) {
682 throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "docId is empty.");
683 }
684 final String docId = docIdObj.toString();
685 final String queryId = request.getParameter("queryId");
686
687 final String[] docIds = userInfoHelper.getResultDocIds(URLDecoder.decode(queryId, Constants.UTF_8));
688 if (docIds == null) {
689 throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "No searched urls.");
690 }
691
692 searchHelper
693 .getDocumentByDocId(docId, new String[] { fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldLang() },
694 OptionalThing.empty())
695 .ifPresent(doc -> {
696 final String favoriteUrl = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
697 final String userCode = userInfoHelper.getUserCode();
698
699 if (StringUtil.isBlank(userCode)) {
700 throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "No user session.");
701 }
702 if (StringUtil.isBlank(favoriteUrl)) {
703 throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "URL is null.");
704 }
705
706 boolean found = false;
707 for (final String id : docIds) {
708 if (docId.equals(id)) {
709 found = true;
710 break;
711 }
712 }
713 if (!found) {
714 throw new WebApiException(HttpServletResponse.SC_NOT_FOUND, "Not found: " + favoriteUrl);
715 }
716
717 if (!favoriteLogService.addUrl(userCode, (userInfo, favoriteLog) -> {
718 favoriteLog.setUserInfoId(userInfo.getId());
719 favoriteLog.setUrl(favoriteUrl);
720 favoriteLog.setDocId(docId);
721 favoriteLog.setQueryId(queryId);
722 favoriteLog.setCreatedAt(systemHelper.getCurrentTimeAsLocalDateTime());
723 })) {
724 throw new WebApiException(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Failed to add url: " + favoriteUrl);
725 }
726
727 final String id = DocumentUtil.getValue(doc, fessConfig.getIndexFieldId(), String.class);
728 searchHelper.update(id, builder -> {
729 final Script script = ComponentUtil.getLanguageHelper()
730 .createScript(doc, "ctx._source." + fessConfig.getIndexFieldFavoriteCount() + "+=1");
731 builder.setScript(script);
732 final Map<String, Object> upsertMap = new HashMap<>();
733 upsertMap.put(fessConfig.getIndexFieldFavoriteCount(), 1);
734 builder.setUpsert(upsertMap);
735 builder.setRefreshPolicy(Constants.TRUE);
736 });
737
738 writeJsonResponse(HttpServletResponse.SC_CREATED, escapeJsonKeyValue(RESULT_FIELD, "created"));
739
740 })
741 .orElse(() -> {
742 throw new WebApiException(HttpServletResponse.SC_NOT_FOUND, "Not found: " + docId);
743 });
744
745 } catch (final WebApiException e) {
746 if (logger.isDebugEnabled()) {
747 logger.debug("Failed to process a favorite request.", e);
748 }
749 writeJsonResponse(e.getStatusCode(), e);
750 } catch (final Exception e) {
751 if (logger.isDebugEnabled()) {
752 logger.debug("Failed to process a favorite request.", e);
753 }
754 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
755 }
756
757 }
758
759
760
761
762
763
764
765 protected void processFavoritesRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain) {
766 if (!acceptHttpMethod(request, GET)) {
767 return;
768 }
769
770 if (!ComponentUtil.getFessConfig().isUserFavorite()) {
771 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, escapeJsonKeyValue(MESSAGE_FIELD, "Unsupported operation."));
772 return;
773 }
774
775 final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
776 final FessConfig fessConfig = ComponentUtil.getFessConfig();
777 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
778 final FavoriteLogService favoriteLogService = ComponentUtil.getComponent(FavoriteLogService.class);
779
780 String body = null;
781 try {
782 final String queryId = request.getParameter("queryId");
783 final String userCode = userInfoHelper.getUserCode();
784
785 if (StringUtil.isBlank(userCode)) {
786 throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "No user session.");
787 }
788 if (StringUtil.isBlank(queryId)) {
789 throw new WebApiException(HttpServletResponse.SC_BAD_REQUEST, "Query ID is null.");
790 }
791
792 final String[] docIds = userInfoHelper.getResultDocIds(queryId);
793 final List<Map<String, Object>> docList = searchHelper.getDocumentListByDocIds(docIds, new String[] {
794 fessConfig.getIndexFieldUrl(), fessConfig.getIndexFieldDocId(), fessConfig.getIndexFieldFavoriteCount() },
795 OptionalThing.empty(), SearchRequestType.JSON);
796 List<String> urlList = new ArrayList<>(docList.size());
797 for (final Map<String, Object> doc : docList) {
798 final String urlObj = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
799 if (urlObj != null) {
800 urlList.add(urlObj);
801 }
802 }
803 urlList = favoriteLogService.getUrlList(userCode, urlList);
804 final List<String> docIdList = new ArrayList<>(urlList.size());
805 for (final Map<String, Object> doc : docList) {
806 final String urlObj = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
807 if (urlObj != null && urlList.contains(urlObj)) {
808 final String docIdObj = DocumentUtil.getValue(doc, fessConfig.getIndexFieldDocId(), String.class);
809 if (docIdObj != null) {
810 docIdList.add(docIdObj);
811 }
812 }
813 }
814
815 final StringBuilder buf = new StringBuilder(255);
816 buf.append("\"record_count\":").append(docIdList.size());
817 buf.append(", \"data\":[");
818 if (!docIdList.isEmpty()) {
819 for (int i = 0; i < docIdList.size(); i++) {
820 if (i > 0) {
821 buf.append(',');
822 }
823 buf.append('{').append(escapeJsonKeyValue(DOC_ID_FIELD, docIdList.get(i))).append('}');
824 }
825 }
826 buf.append(']');
827 body = buf.toString();
828 writeJsonResponse(HttpServletResponse.SC_OK, body);
829 } catch (final WebApiException e) {
830 if (logger.isDebugEnabled()) {
831 logger.debug("Failed to process a favorites request.", e);
832 }
833 writeJsonResponse(e.getStatusCode(), e);
834 } catch (final Exception e) {
835 if (logger.isDebugEnabled()) {
836 logger.debug("Failed to process a favorites request.", e);
837 }
838 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
839 }
840 }
841
842
843
844
845
846
847
848
849
850 protected void processSuggestRequest(final HttpServletRequest request, final HttpServletResponse response, final FilterChain chain)
851 throws IOException, ServletException {
852 if (!acceptHttpMethod(request, GET)) {
853 return;
854 }
855
856 final FessConfig fessConfig = ComponentUtil.getFessConfig();
857 if (!fessConfig.isAcceptedSearchReferer(request.getHeader("referer"))) {
858 writeJsonResponse(HttpServletResponse.SC_BAD_REQUEST, escapeJsonKeyValue(MESSAGE_FIELD, "Referer is invalid."));
859 return;
860 }
861
862 final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
863 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
864
865 final StringBuilder buf = new StringBuilder(255);
866 try {
867 final RequestParameter parameter = RequestParameter.parse(request);
868 final String[] langs = searchHelper.getLanguages(request, parameter);
869
870 final SuggestHelper suggestHelper = ComponentUtil.getSuggestHelper();
871 final SuggestRequestBuilder builder = suggestHelper.suggester().suggest();
872 builder.setQuery(parameter.getQuery());
873 stream(parameter.getSuggestFields()).of(stream -> stream.forEach(builder::addField));
874 roleQueryHelper.build(SearchRequestType.SUGGEST).stream().forEach(builder::addRole);
875 builder.setSize(parameter.getNum());
876 stream(langs).of(stream -> stream.forEach(builder::addLang));
877
878 stream(parameter.getTags()).of(stream -> stream.forEach(builder::addTag));
879 final String key = ComponentUtil.getVirtualHostHelper().getVirtualHostKey();
880 if (StringUtil.isNotBlank(key)) {
881 builder.addTag(key);
882 }
883
884 builder.addKind(SuggestItem.Kind.USER.toString());
885 if (ComponentUtil.getFessConfig().isSuggestSearchLog()) {
886 builder.addKind(SuggestItem.Kind.QUERY.toString());
887 }
888 if (ComponentUtil.getFessConfig().isSuggestDocuments()) {
889 builder.addKind(SuggestItem.Kind.DOCUMENT.toString());
890 }
891
892 final SuggestResponse suggestResponse = builder.execute().getResponse();
893
894 buf.append("\"query_time\":").append(suggestResponse.getTookMs());
895 buf.append(",\"record_count\":").append(suggestResponse.getTotal());
896 buf.append(",\"page_size\":").append(suggestResponse.getNum());
897
898 if (!suggestResponse.getItems().isEmpty()) {
899 buf.append(",\"data\":[");
900
901 boolean first = true;
902 for (final SuggestItem item : suggestResponse.getItems()) {
903 if (!first) {
904 buf.append(',');
905 }
906 first = false;
907
908 buf.append("{\"text\":\"").append(StringEscapeUtils.escapeJson(item.getText())).append('\"');
909 buf.append(",\"labels\":[");
910 for (int i = 0; i < item.getTags().length; i++) {
911 if (i > 0) {
912 buf.append(',');
913 }
914 buf.append('\"').append(StringEscapeUtils.escapeJson(item.getTags()[i])).append('\"');
915 }
916 buf.append(']');
917 buf.append('}');
918 }
919 buf.append(']');
920 }
921
922 writeJsonResponse(HttpServletResponse.SC_OK, buf.toString());
923 } catch (final Exception e) {
924 if (logger.isDebugEnabled()) {
925 logger.debug("Failed to process a suggest request.", e);
926 }
927 writeJsonResponse(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e);
928 }
929 }
930
931
932
933
934 protected static class JsonRequestParams extends SearchRequestParams {
935
936 private final HttpServletRequest request;
937
938 private final FessConfig fessConfig;
939
940 private int startPosition = -1;
941
942 private int offset = -1;
943
944 private int pageSize = -1;
945
946
947
948
949
950
951 protected JsonRequestParams(final HttpServletRequest request, final FessConfig fessConfig) {
952 this.request = request;
953 this.fessConfig = fessConfig;
954 }
955
956 @Override
957 public String getTrackTotalHits() {
958 return request.getParameter(Constants.TRACK_TOTAL_HITS);
959 }
960
961 @Override
962 public String getQuery() {
963 return request.getParameter("q");
964 }
965
966 @Override
967 public String[] getExtraQueries() {
968 return getParamValueArray(request, "ex_q");
969 }
970
971 @Override
972 public Map<String, String[]> getFields() {
973 final Map<String, String[]> fields = new HashMap<>();
974 for (final Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
975 final String key = entry.getKey();
976 if (key.startsWith("fields.")) {
977 final String[] value = simplifyArray(entry.getValue());
978 fields.put(key.substring("fields.".length()), value);
979 }
980 }
981 return fields;
982 }
983
984 @Override
985 public Map<String, String[]> getConditions() {
986 final Map<String, String[]> conditions = new HashMap<>();
987 for (final Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
988 final String key = entry.getKey();
989 if (key.startsWith("as.")) {
990 final String[] value = simplifyArray(entry.getValue());
991 conditions.put(key.substring("as.".length()), value);
992 }
993 }
994 return conditions;
995 }
996
997 @Override
998 public String[] getLanguages() {
999 return getParamValueArray(request, "lang");
1000 }
1001
1002 @Override
1003 public GeoInfo getGeoInfo() {
1004 return createGeoInfo(request);
1005 }
1006
1007 @Override
1008 public FacetInfo getFacetInfo() {
1009 return createFacetInfo(request);
1010 }
1011
1012 @Override
1013 public String getSort() {
1014 return request.getParameter("sort");
1015 }
1016
1017 @Override
1018 public int getStartPosition() {
1019 if (startPosition != -1) {
1020 return startPosition;
1021 }
1022
1023 final String start = request.getParameter("start");
1024 if (StringUtil.isBlank(start)) {
1025 startPosition = fessConfig.getPagingSearchPageStartAsInteger();
1026 } else {
1027 try {
1028 startPosition = Integer.parseInt(start);
1029 } catch (final NumberFormatException e) {
1030 startPosition = fessConfig.getPagingSearchPageStartAsInteger();
1031 }
1032 }
1033 return startPosition;
1034 }
1035
1036 @Override
1037 public int getOffset() {
1038 if (offset != -1) {
1039 return offset;
1040 }
1041
1042 final String value = request.getParameter("offset");
1043 if (StringUtil.isBlank(value)) {
1044 offset = 0;
1045 } else {
1046 try {
1047 offset = Integer.parseInt(value);
1048 } catch (final NumberFormatException e) {
1049 offset = 0;
1050 }
1051 }
1052 return offset;
1053 }
1054
1055 @Override
1056 public int getPageSize() {
1057 if (pageSize != -1) {
1058 return pageSize;
1059 }
1060
1061 final String num = request.getParameter("num");
1062 if (StringUtil.isBlank(num)) {
1063 pageSize = fessConfig.getPagingSearchPageSizeAsInteger();
1064 } else {
1065 try {
1066 pageSize = Integer.parseInt(num);
1067 if (pageSize > fessConfig.getPagingSearchPageMaxSizeAsInteger().intValue() || pageSize <= 0) {
1068 pageSize = fessConfig.getPagingSearchPageMaxSizeAsInteger();
1069 }
1070 } catch (final NumberFormatException e) {
1071 pageSize = fessConfig.getPagingSearchPageSizeAsInteger();
1072 }
1073 }
1074 return pageSize;
1075 }
1076
1077 @Override
1078 public Object getAttribute(final String name) {
1079 return request.getAttribute(name);
1080 }
1081
1082 @Override
1083 public Locale getLocale() {
1084 return Locale.ROOT;
1085 }
1086
1087 @Override
1088 public SearchRequestType getType() {
1089 return SearchRequestType.JSON;
1090 }
1091
1092 @Override
1093 public String getSimilarDocHash() {
1094 return request.getParameter("sdh");
1095 }
1096
1097 @Override
1098 public HighlightInfo getHighlightInfo() {
1099 return ComponentUtil.getViewHelper().createHighlightInfo();
1100 }
1101 }
1102
1103
1104
1105
1106 protected static class RequestParameter extends SearchRequestParams {
1107 private final String query;
1108
1109 private final String[] fields;
1110
1111 private final int num;
1112
1113 private final HttpServletRequest request;
1114
1115 private final String[] tags;
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125 protected RequestParameter(final HttpServletRequest request, final String query, final String[] tags, final String[] fields,
1126 final int num) {
1127 this.query = query;
1128 this.tags = tags;
1129 this.fields = fields;
1130 this.num = num;
1131 this.request = request;
1132 }
1133
1134
1135
1136
1137
1138
1139 protected static RequestParameter parse(final HttpServletRequest request) {
1140 final String query = request.getParameter("q");
1141 final String[] tags = getParamValueArray(request, "label");
1142 final String[] fields = getParamValueArray(request, "field");
1143
1144 final String numStr = request.getParameter("num");
1145 final int num;
1146 if (StringUtil.isNotBlank(numStr) && StringUtils.isNumeric(numStr)) {
1147 num = Integer.parseInt(numStr);
1148 } else {
1149 num = 10;
1150 }
1151
1152 return new RequestParameter(request, query, tags, fields, num);
1153 }
1154
1155 @Override
1156 public String getQuery() {
1157 return query;
1158 }
1159
1160
1161
1162
1163
1164 protected String[] getSuggestFields() {
1165 return fields;
1166 }
1167
1168
1169
1170
1171
1172 protected int getNum() {
1173 return num;
1174 }
1175
1176 @Override
1177 public Map<String, String[]> getFields() {
1178 return Collections.emptyMap();
1179 }
1180
1181 @Override
1182 public Map<String, String[]> getConditions() {
1183 return Collections.emptyMap();
1184 }
1185
1186
1187
1188
1189
1190 public String[] getTags() {
1191 return tags;
1192 }
1193
1194 @Override
1195 public String[] getLanguages() {
1196 return getParamValueArray(request, "lang");
1197 }
1198
1199 @Override
1200 public GeoInfo getGeoInfo() {
1201 throw new UnsupportedOperationException("getGeoInfo() is not supported in this implementation");
1202 }
1203
1204 @Override
1205 public FacetInfo getFacetInfo() {
1206 throw new UnsupportedOperationException("getFacetInfo() is not supported in this implementation");
1207 }
1208
1209 @Override
1210 public String getSort() {
1211 throw new UnsupportedOperationException("getSort() is not supported in this implementation");
1212 }
1213
1214 @Override
1215 public int getStartPosition() {
1216 throw new UnsupportedOperationException("getStartPosition() is not supported in this implementation");
1217 }
1218
1219 @Override
1220 public int getOffset() {
1221 throw new UnsupportedOperationException("getOffset() is not supported in this implementation");
1222 }
1223
1224 @Override
1225 public int getPageSize() {
1226 throw new UnsupportedOperationException("getPageSize() is not supported in this implementation");
1227 }
1228
1229 @Override
1230 public String[] getExtraQueries() {
1231 throw new UnsupportedOperationException("getExtraQueries() is not supported in this implementation");
1232 }
1233
1234 @Override
1235 public Object getAttribute(final String name) {
1236 throw new UnsupportedOperationException("getAttribute() is not supported in this implementation");
1237 }
1238
1239 @Override
1240 public Locale getLocale() {
1241 throw new UnsupportedOperationException("getLocale() is not supported in this implementation");
1242 }
1243
1244 @Override
1245 public SearchRequestType getType() {
1246 return SearchRequestType.SUGGEST;
1247 }
1248
1249 @Override
1250 public String getSimilarDocHash() {
1251 throw new UnsupportedOperationException("getSimilarDocHash() is not supported in this implementation");
1252 }
1253
1254 @Override
1255 public HighlightInfo getHighlightInfo() {
1256 return new HighlightInfo();
1257 }
1258 }
1259
1260 @Override
1261 protected void writeHeaders(final HttpServletResponse response) {
1262 ComponentUtil.getFessConfig().getApiJsonResponseHeaderList().forEach(e -> response.setHeader(e.getFirst(), e.getSecond()));
1263 }
1264
1265
1266
1267
1268
1269
1270 protected void writeJsonResponse(final int status, final Throwable t) {
1271 final Supplier<String> stacktraceString = () -> {
1272 final StringBuilder buf = new StringBuilder(100);
1273 if (StringUtil.isBlank(t.getMessage())) {
1274 buf.append(t.getClass().getName());
1275 } else {
1276 buf.append(t.getMessage());
1277 }
1278 try (final StringWriter sw = new StringWriter(); final PrintWriter pw = new PrintWriter(sw)) {
1279 t.printStackTrace(pw);
1280 pw.flush();
1281 buf.append(" [ ").append(sw.toString()).append(" ]");
1282 } catch (final IOException e) {
1283
1284 if (logger.isDebugEnabled()) {
1285 logger.debug("Unexpected IOException while closing StringWriter", e);
1286 }
1287 }
1288 return buf.toString();
1289 };
1290 final String message;
1291 if (Constants.TRUE.equalsIgnoreCase(ComponentUtil.getFessConfig().getApiJsonResponseExceptionIncluded())) {
1292 message = escapeJsonKeyValue(MESSAGE_FIELD, stacktraceString.get());
1293 } else {
1294 final String errorCode = UUID.randomUUID().toString();
1295 message = escapeJsonKeyValue("error_code:", errorCode);
1296 if (logger.isDebugEnabled()) {
1297 logger.debug("[{}] {}", errorCode, stacktraceString.get().replace("\n", "\\n"));
1298 } else if (status >= 500) {
1299 logger.warn("[{}] {}", errorCode, t.getMessage());
1300 }
1301 }
1302 final HttpServletResponse response = LaResponseUtil.getResponse();
1303 if (t instanceof final InvalidAccessTokenException e) {
1304 response.setHeader("WWW-Authenticate", "Bearer error=\"" + e.getType() + "\"");
1305 writeJsonResponse(HttpServletResponse.SC_UNAUTHORIZED, message);
1306 } else {
1307 writeJsonResponse(status, message);
1308 }
1309 }
1310
1311
1312
1313
1314
1315
1316
1317 protected String escapeJsonKeyValue(final String key, final String value) {
1318 return "\"" + key + "\":" + escapeJson(value);
1319 }
1320
1321
1322
1323
1324
1325
1326 protected void writeJsonResponse(final int status, final String body) {
1327 final String callback = LaRequestUtil.getOptionalRequest().map(req -> req.getParameter("callback")).orElse(null);
1328 final boolean isJsonp = ComponentUtil.getFessConfig().isApiJsonpEnabled() && StringUtil.isNotBlank(callback);
1329
1330 final HttpServletResponse response = LaResponseUtil.getResponse();
1331 response.setStatus(status);
1332
1333 final StringBuilder buf = new StringBuilder(1000);
1334 if (isJsonp) {
1335 buf.append(escapeCallbackName(callback));
1336 buf.append('(');
1337 }
1338 buf.append('{');
1339 if (StringUtil.isNotBlank(body)) {
1340 buf.append(body);
1341 }
1342 buf.append('}');
1343 if (isJsonp) {
1344 buf.append(')');
1345 }
1346 write(buf.toString(), mimeType, Constants.UTF_8);
1347 }
1348
1349
1350
1351
1352
1353
1354 protected String escapeCallbackName(final String callbackName) {
1355 return "/**/" + callbackName.replaceAll("[^0-9a-zA-Z_\\$\\.]", StringUtil.EMPTY);
1356 }
1357
1358
1359
1360
1361
1362
1363 protected String escapeJson(final Object obj) {
1364 if (obj == null) {
1365 return "null";
1366 }
1367
1368 final StringBuilder buf = new StringBuilder(255);
1369 if (obj instanceof String[]) {
1370 buf.append('[');
1371 boolean first = true;
1372 for (final Object child : (String[]) obj) {
1373 if (first) {
1374 first = false;
1375 } else {
1376 buf.append(',');
1377 }
1378 buf.append(escapeJson(child));
1379 }
1380 buf.append(']');
1381 } else if (obj instanceof List<?>) {
1382 buf.append('[');
1383 boolean first = true;
1384 for (final Object child : (List<?>) obj) {
1385 if (first) {
1386 first = false;
1387 } else {
1388 buf.append(',');
1389 }
1390 buf.append(escapeJson(child));
1391 }
1392 buf.append(']');
1393 } else if (obj instanceof Map<?, ?>) {
1394 buf.append('{');
1395 boolean first = true;
1396 for (final Map.Entry<?, ?> entry : ((Map<?, ?>) obj).entrySet()) {
1397 if (first) {
1398 first = false;
1399 } else {
1400 buf.append(',');
1401 }
1402 buf.append(escapeJson(entry.getKey())).append(':').append(escapeJson(entry.getValue()));
1403 }
1404 buf.append('}');
1405 } else if (obj instanceof Integer || obj instanceof Long || obj instanceof Float || obj instanceof Double) {
1406 buf.append(obj);
1407 } else if (obj instanceof Boolean) {
1408 buf.append(obj.toString());
1409 } else if (obj instanceof Date) {
1410 final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND, Locale.ROOT);
1411 buf.append('\"').append(StringEscapeUtils.escapeJson(sdf.format(obj))).append('\"');
1412 } else {
1413 buf.append('\"').append(StringEscapeUtils.escapeJson(obj.toString())).append('\"');
1414 }
1415 return buf.toString();
1416 }
1417
1418
1419
1420
1421
1422 public void setMimeType(final String mimeType) {
1423 this.mimeType = mimeType;
1424 }
1425
1426 }