1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package org.codelibs.fess.helper;
17
18 import static org.codelibs.core.stream.StreamUtil.split;
19
20 import java.io.BufferedInputStream;
21 import java.io.File;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.UnsupportedEncodingException;
25 import java.net.URLDecoder;
26 import java.net.URLEncoder;
27 import java.util.ArrayList;
28 import java.util.Date;
29 import java.util.HashMap;
30 import java.util.HashSet;
31 import java.util.List;
32 import java.util.Locale;
33 import java.util.Map;
34 import java.util.Set;
35 import java.util.concurrent.ConcurrentHashMap;
36 import java.util.concurrent.ExecutionException;
37 import java.util.concurrent.TimeUnit;
38 import java.util.function.Consumer;
39 import java.util.function.Function;
40 import java.util.regex.Matcher;
41 import java.util.regex.Pattern;
42 import java.util.stream.Collectors;
43
44 import javax.annotation.PostConstruct;
45 import javax.servlet.ServletContext;
46 import javax.servlet.SessionTrackingMode;
47 import javax.servlet.http.HttpServletRequest;
48 import javax.servlet.http.HttpSession;
49
50 import org.apache.catalina.connector.ClientAbortException;
51 import org.apache.commons.lang3.StringUtils;
52 import org.apache.commons.text.StringEscapeUtils;
53 import org.apache.logging.log4j.LogManager;
54 import org.apache.logging.log4j.Logger;
55 import org.codelibs.core.CoreLibConstants;
56 import org.codelibs.core.io.CloseableUtil;
57 import org.codelibs.core.lang.StringUtil;
58 import org.codelibs.core.misc.DynamicProperties;
59 import org.codelibs.core.stream.StreamUtil;
60 import org.codelibs.fesen.common.text.Text;
61 import org.codelibs.fesen.search.fetch.subphase.highlight.HighlightField;
62 import org.codelibs.fess.Constants;
63 import org.codelibs.fess.app.web.base.SearchForm;
64 import org.codelibs.fess.app.web.base.login.FessLoginAssist;
65 import org.codelibs.fess.crawler.builder.RequestDataBuilder;
66 import org.codelibs.fess.crawler.client.CrawlerClient;
67 import org.codelibs.fess.crawler.client.CrawlerClientFactory;
68 import org.codelibs.fess.crawler.entity.ResponseData;
69 import org.codelibs.fess.crawler.util.CharUtil;
70 import org.codelibs.fess.entity.FacetQueryView;
71 import org.codelibs.fess.entity.HighlightInfo;
72 import org.codelibs.fess.entity.SearchRenderData;
73 import org.codelibs.fess.es.config.exentity.CrawlingConfig;
74 import org.codelibs.fess.exception.FessSystemException;
75 import org.codelibs.fess.helper.UserAgentHelper.UserAgentType;
76 import org.codelibs.fess.mylasta.action.FessUserBean;
77 import org.codelibs.fess.mylasta.direction.FessConfig;
78 import org.codelibs.fess.util.ComponentUtil;
79 import org.codelibs.fess.util.DocumentUtil;
80 import org.codelibs.fess.util.FacetResponse;
81 import org.codelibs.fess.util.ResourceUtil;
82 import org.dbflute.optional.OptionalThing;
83 import org.lastaflute.taglib.function.LaFunctions;
84 import org.lastaflute.web.response.ActionResponse;
85 import org.lastaflute.web.response.StreamResponse;
86 import org.lastaflute.web.ruts.process.ActionRuntime;
87 import org.lastaflute.web.util.LaRequestUtil;
88 import org.lastaflute.web.util.LaResponseUtil;
89 import org.lastaflute.web.util.LaServletContextUtil;
90
91 import com.github.jknack.handlebars.Context;
92 import com.github.jknack.handlebars.Handlebars;
93 import com.github.jknack.handlebars.Template;
94 import com.github.jknack.handlebars.io.FileTemplateLoader;
95 import com.google.common.cache.Cache;
96 import com.google.common.cache.CacheBuilder;
97 import com.ibm.icu.text.SimpleDateFormat;
98
99 public class ViewHelper {
100
101 private static final Logger logger = LogManager.getLogger(ViewHelper.class);
102
103 protected static final String SCREEN_WIDTH = "screen_width";
104
105 protected static final int TABLET_WIDTH = 768;
106
107 protected static final String CONTENT_DISPOSITION = "Content-Disposition";
108
109 protected static final String HL_CACHE = "hl_cache";
110
111 protected static final String QUERIES = "queries";
112
113 protected static final String CACHE_MSG = "cache_msg";
114
115 protected static final Pattern LOCAL_PATH_PATTERN = Pattern.compile("^file:/+[a-zA-Z]:");
116
117 protected static final Pattern SHARED_FOLDER_PATTERN = Pattern.compile("^file:/+[^/]\\.");
118
119 protected static final String ELLIPSIS = "...";
120
121 protected boolean encodeUrlLink = false;
122
123 protected String urlLinkEncoding = Constants.UTF_8;
124
125 protected String[] highlightedFields;
126
127 protected String originalHighlightTagPre = "<em>";
128
129 protected String originalHighlightTagPost = "</em>";
130
131 protected String highlightTagPre;
132
133 protected String highlightTagPost;
134
135 protected boolean useSession = true;
136
137 protected final Map<String, String> pageCacheMap = new ConcurrentHashMap<>();
138
139 protected final Map<String, String> initFacetParamMap = new HashMap<>();
140
141 protected final Map<String, String> initGeoParamMap = new HashMap<>();
142
143 protected final List<FacetQueryView> facetQueryViewList = new ArrayList<>();
144
145 protected String cacheTemplateName = "cache";
146
147 protected String escapedHighlightPre = null;
148
149 protected String escapedHighlightPost = null;
150
151 protected Set<Integer> highlightTerminalCharSet = new HashSet<>();
152
153 protected ActionHook actionHook = new ActionHook();
154
155 protected final Set<String> inlineMimeTypeSet = new HashSet<>();
156
157 protected Cache<String, FacetResponse> facetCache;
158
159 protected long facetCacheDuration = 60 * 10L;
160
161 protected int textFragmentPrefixLength;
162
163 protected int textFragmentSuffixLength;
164
165 protected int textFragmentSize;
166
167 @PostConstruct
168 public void init() {
169 if (logger.isDebugEnabled()) {
170 logger.debug("Initialize {}", this.getClass().getSimpleName());
171 }
172 final FessConfig fessConfig = ComponentUtil.getFessConfig();
173 escapedHighlightPre = LaFunctions.h(originalHighlightTagPre);
174 escapedHighlightPost = LaFunctions.h(originalHighlightTagPost);
175 highlightTagPre = fessConfig.getQueryHighlightTagPre();
176 highlightTagPost = fessConfig.getQueryHighlightTagPost();
177 highlightedFields = fessConfig.getQueryHighlightContentDescriptionFieldsAsArray();
178 for (final int v : fessConfig.getQueryHighlightTerminalCharsAsArray()) {
179 highlightTerminalCharSet.add(v);
180 }
181 try {
182 final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
183 servletContext.setSessionTrackingModes(
184 fessConfig.getSessionTrackingModesAsSet().stream().map(SessionTrackingMode::valueOf).collect(Collectors.toSet()));
185 } catch (final Throwable t) {
186 logger.warn("Failed to set SessionTrackingMode.", t);
187 }
188
189 split(fessConfig.getQueryFacetQueries(), "\n").of(stream -> stream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(s -> {
190 final String[] values = StringUtils.split(s, ":", 2);
191 if (values.length != 2) {
192 return;
193 }
194 final FacetQueryView facetQueryView = new FacetQueryView();
195 facetQueryView.setTitle(values[0]);
196 split(values[1], "\t").of(subStream -> subStream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(v -> {
197 final String[] facet = StringUtils.split(v, "=", 2);
198 if (facet.length == 2) {
199 facetQueryView.addQuery(facet[0], facet[1]);
200 }
201 }));
202 facetQueryView.init();
203 facetQueryViewList.add(facetQueryView);
204 if (logger.isDebugEnabled()) {
205 logger.debug("loaded {}", facetQueryView);
206 }
207 }));
208
209 facetCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(facetCacheDuration, TimeUnit.SECONDS).build();
210
211 textFragmentPrefixLength = fessConfig.getQueryHighlightTextFragmentPrefixLengthAsInteger();
212 textFragmentSuffixLength = fessConfig.getQueryHighlightTextFragmentSuffixLengthAsInteger();
213 textFragmentSize = fessConfig.getQueryHighlightTextFragmentSizeAsInteger();
214 }
215
216 public String getContentTitle(final Map<String, Object> document) {
217 final FessConfig fessConfig = ComponentUtil.getFessConfig();
218 String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
219 if (StringUtil.isBlank(title)) {
220 title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
221 if (StringUtil.isBlank(title)) {
222 title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
223 }
224 }
225 final int size = fessConfig.getResponseMaxTitleLengthAsInteger();
226 if (size > -1) {
227 title = StringUtils.abbreviate(title, size);
228 }
229 final String value = LaFunctions.h(title);
230 if (!fessConfig.isResponseHighlightContentTitleEnabled()) {
231 return value;
232 }
233 return getQuerySet().map(querySet -> {
234 final String pattern = querySet.stream().map(LaFunctions::h).map(Pattern::quote).collect(Collectors.joining("|"));
235 if (StringUtil.isBlank(pattern)) {
236 return null;
237 }
238 final Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE).matcher(value);
239 final StringBuffer buf = new StringBuffer(value.length() + 100);
240 while (matcher.find()) {
241 matcher.appendReplacement(buf, highlightTagPre + matcher.group(0) + highlightTagPost);
242 }
243 matcher.appendTail(buf);
244 return buf.toString();
245 }).orElse(value);
246 }
247
248 protected OptionalThing<Set<String>> getQuerySet() {
249 return LaRequestUtil.getOptionalRequest().map(req -> {
250 @SuppressWarnings("unchecked")
251 final Set<String> querySet = (Set<String>) req.getAttribute(Constants.HIGHLIGHT_QUERIES);
252 return querySet;
253 }).filter(s -> s != null);
254 }
255
256 public String getContentDescription(final Map<String, Object> document) {
257 for (final String field : highlightedFields) {
258 final String text = DocumentUtil.getValue(document, field, String.class);
259 if (StringUtil.isNotBlank(text)) {
260 return escapeHighlight(text);
261 }
262 }
263
264 return StringUtil.EMPTY;
265 }
266
267 protected String escapeHighlight(final String text) {
268 final String escaped = LaFunctions.h(text);
269 final String value;
270 if (ComponentUtil.getFessConfig().isQueryHighlightBoundaryPositionDetect()) {
271 int pos = escaped.indexOf(escapedHighlightPre);
272 while (pos >= 0) {
273 final int c = escaped.codePointAt(pos);
274 if (Character.isISOControl(c) || highlightTerminalCharSet.contains(c)) {
275 break;
276 }
277 pos--;
278 }
279
280 value = escaped.substring(pos + 1);
281 } else {
282 value = escaped;
283 }
284 return value.replaceAll(escapedHighlightPre, highlightTagPre).replaceAll(escapedHighlightPost, highlightTagPost);
285 }
286
287 protected String removeHighlightTag(final String str) {
288 return str.replaceAll(originalHighlightTagPre, StringUtil.EMPTY).replaceAll(originalHighlightTagPost, StringUtil.EMPTY);
289 }
290
291 public HighlightInfo createHighlightInfo() {
292 return LaRequestUtil.getOptionalRequest().map(req -> {
293 final HighlightInfo highlightInfo = new HighlightInfo();
294 final String widthStr = req.getParameter(SCREEN_WIDTH);
295 if (StringUtil.isNotBlank(widthStr)) {
296 final int width = Integer.parseInt(widthStr);
297 updateHighlightInfo(highlightInfo, width);
298 final HttpSession session = req.getSession(false);
299 if (session != null) {
300 session.setAttribute(SCREEN_WIDTH, width);
301 }
302 } else {
303 final HttpSession session = req.getSession(false);
304 if (session != null) {
305 final Integer width = (Integer) session.getAttribute(SCREEN_WIDTH);
306 if (width != null) {
307 updateHighlightInfo(highlightInfo, width);
308 }
309 }
310 }
311 return highlightInfo;
312 }).orElse(new HighlightInfo());
313 }
314
315 protected void updateHighlightInfo(final HighlightInfo highlightInfo, final int width) {
316 if (width < TABLET_WIDTH) {
317 float ratio = ((float) width) / ((float) TABLET_WIDTH);
318 if (ratio < 0.5) {
319 ratio = 0.5f;
320 }
321 highlightInfo.fragmentSize((int) (highlightInfo.getFragmentSize() * ratio));
322 }
323 }
324
325 public String getUrlLink(final Map<String, Object> document) {
326 final FessConfig fessConfig = ComponentUtil.getFessConfig();
327 String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
328
329 if (StringUtil.isBlank(url)) {
330 return "#not-found-" + DocumentUtil.getValue(document, fessConfig.getIndexFieldDocId(), String.class);
331 }
332
333 final boolean isSmbUrl = url.startsWith("smb:") || url.startsWith("smb1:");
334 final boolean isFtpUrl = url.startsWith("ftp:");
335 final boolean isSmbOrFtpUrl = isSmbUrl || isFtpUrl;
336
337
338 url = ComponentUtil.getPathMappingHelper().replaceUrl(url);
339
340 final boolean isHttpUrl = url.startsWith("http:") || url.startsWith("https:");
341
342 if (isSmbUrl) {
343 url = url.replace("smb:", "file:");
344 url = url.replace("smb1:", "file:");
345 }
346
347 if (isHttpUrl && isSmbOrFtpUrl) {
348
349
350 final StringBuilder buf = new StringBuilder(url.length() + 100);
351 for (final char c : url.toCharArray()) {
352 if (CharUtil.isUrlChar(c)) {
353 buf.append(c);
354 } else {
355 try {
356 buf.append(URLEncoder.encode(String.valueOf(c), urlLinkEncoding));
357 } catch (final UnsupportedEncodingException e) {
358 buf.append(c);
359 }
360 }
361 }
362 url = buf.toString();
363 } else if (url.startsWith("file:")) {
364
365 url = updateFileProtocol(url);
366
367 if (encodeUrlLink) {
368 return appendQueryParameter(document, url);
369 }
370
371
372 if (!isSmbOrFtpUrl) {
373
374 try {
375 url = URLDecoder.decode(url.replace("+", "%2B"), urlLinkEncoding);
376 } catch (final Exception e) {
377 if (logger.isDebugEnabled()) {
378 logger.warn("Failed to decode {}", url, e);
379 }
380 }
381 }
382 }
383
384
385
386 return appendQueryParameter(document, url);
387 }
388
389 protected String updateFileProtocol(String url) {
390 final int pos = url.indexOf(':', 5);
391 final boolean isLocalFile = pos > 0 && pos < 12;
392
393 final UserAgentType ua = ComponentUtil.getUserAgentHelper().getUserAgentType();
394 final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
395 switch (ua) {
396 case IE:
397 if (isLocalFile) {
398 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.ie", "file://"));
399 } else {
400 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.ie", "file://"));
401 }
402 break;
403 case FIREFOX:
404 if (isLocalFile) {
405 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.firefox", "file://"));
406 } else {
407 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.firefox", "file://///"));
408 }
409 break;
410 case CHROME:
411 if (isLocalFile) {
412 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.chrome", "file://"));
413 } else {
414 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.chrome", "file://"));
415 }
416 break;
417 case SAFARI:
418 if (isLocalFile) {
419 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.safari", "file://"));
420 } else {
421 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.safari", "file:////"));
422 }
423 break;
424 case OPERA:
425 if (isLocalFile) {
426 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.opera", "file://"));
427 } else {
428 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.opera", "file://"));
429 }
430 break;
431 default:
432 if (isLocalFile) {
433 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.other", "file://"));
434 } else {
435 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.other", "file://"));
436 }
437 break;
438 }
439 return url;
440 }
441
442 protected String appendQueryParameter(final Map<String, Object> document, final String url) {
443 final FessConfig fessConfig = ComponentUtil.getFessConfig();
444 if (fessConfig.isAppendQueryParameter()) {
445 if (url.indexOf('#') >= 0) {
446 return url;
447 }
448
449 final String mimetype = DocumentUtil.getValue(document, fessConfig.getIndexFieldMimetype(), String.class);
450 if (StringUtil.isNotBlank(mimetype)) {
451 switch (mimetype) {
452 case "text/html":
453 return appendHTMLSearchWord(document, url);
454 case "application/pdf":
455 return appendPDFSearchWord(document, url);
456 default:
457 break;
458 }
459 }
460 }
461 return url;
462 }
463
464 protected String appendHTMLSearchWord(final Map<String, Object> document, final String url) {
465 final TextFragment[] textFragments = (TextFragment[]) document.get(Constants.TEXT_FRAGMENTS);
466 if (textFragments != null) {
467 final StringBuilder buf = new StringBuilder(1000);
468 buf.append(url).append("#:~:");
469 for (int i = 0; i < textFragmentSize && i < textFragments.length; i++) {
470 buf.append(textFragments[i].toURLString()).append('&');
471 }
472 return buf.toString();
473 }
474 return url;
475 }
476
477 protected String appendPDFSearchWord(final Map<String, Object> document, final String url) {
478 final String queries = (String) LaRequestUtil.getRequest().getAttribute(Constants.REQUEST_QUERIES);
479 if (queries != null) {
480 try {
481 final StringBuilder buf = new StringBuilder(url.length() + 100);
482 buf.append(url).append("#search=%22");
483 buf.append(URLEncoder.encode(queries.trim(), Constants.UTF_8));
484 buf.append("%22");
485 return buf.toString();
486 } catch (final UnsupportedEncodingException e) {
487 logger.warn("Unsupported encoding.", e);
488 }
489 }
490 return url;
491 }
492
493 public String getPagePath(final String page) {
494 final Locale locale = ComponentUtil.getRequestManager().getUserLocale();
495 final String lang = locale.getLanguage();
496 final String country = locale.getCountry();
497
498 final String pathLC = getLocalizedPagePath(page, lang, country);
499 final String pLC = pageCacheMap.get(pathLC);
500 if (pLC != null) {
501 return pLC;
502 }
503 if (existsPage(pathLC)) {
504 pageCacheMap.put(pathLC, pathLC);
505 return pathLC;
506 }
507
508 final String pathL = getLocalizedPagePath(page, lang, null);
509 final String pL = pageCacheMap.get(pathL);
510 if (pL != null) {
511 return pL;
512 }
513 if (existsPage(pathL)) {
514 pageCacheMap.put(pathLC, pathL);
515 return pathL;
516 }
517
518 final String path = getLocalizedPagePath(page, null, null);
519 final String p = pageCacheMap.get(path);
520 if (p != null) {
521 return p;
522 }
523 if (existsPage(path)) {
524 pageCacheMap.put(pathLC, path);
525 return path;
526 }
527
528 return "index.jsp";
529 }
530
531 private String getLocalizedPagePath(final String page, final String lang, final String country) {
532 final StringBuilder buf = new StringBuilder(100);
533 buf.append("/WEB-INF/view/").append(page);
534 if (StringUtil.isNotBlank(lang)) {
535 buf.append('_').append(lang);
536 if (StringUtil.isNotBlank(country)) {
537 buf.append('_').append(country);
538 }
539 }
540 buf.append(".jsp");
541 return buf.toString();
542 }
543
544 private boolean existsPage(final String path) {
545 final String realPath = LaServletContextUtil.getServletContext().getRealPath(path);
546 final File file = new File(realPath);
547 return file.isFile();
548 }
549
550 public String createCacheContent(final Map<String, Object> doc, final String[] queries) {
551 final FessConfig fessConfig = ComponentUtil.getFessConfig();
552 final FileTemplateLoader loader = new FileTemplateLoader(ResourceUtil.getViewTemplatePath().toFile());
553 final Handlebars handlebars = new Handlebars(loader);
554
555 Locale locale = ComponentUtil.getRequestManager().getUserLocale();
556 if (locale == null) {
557 locale = Locale.ENGLISH;
558 }
559 String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
560 if (url == null) {
561 url = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
562 }
563 doc.put(fessConfig.getResponseFieldUrlLink(), getUrlLink(doc));
564 String createdStr;
565 final Date created = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCreated(), Date.class);
566 if (created != null) {
567 final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
568 createdStr = sdf.format(created);
569 } else {
570 createdStr = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
571 }
572 doc.put(CACHE_MSG, ComponentUtil.getMessageManager().getMessage(locale, "labels.search_cache_msg", url, createdStr));
573
574 doc.put(QUERIES, queries);
575
576 String cache = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCache(), String.class);
577 if (cache != null) {
578 final String mimetype = DocumentUtil.getValue(doc, fessConfig.getIndexFieldMimetype(), String.class);
579 if (!ComponentUtil.getFessConfig().isHtmlMimetypeForCache(mimetype)) {
580 cache = StringEscapeUtils.escapeHtml4(cache);
581 }
582 cache = ComponentUtil.getPathMappingHelper().replaceUrls(cache);
583 if (queries != null && queries.length > 0) {
584 doc.put(HL_CACHE, replaceHighlightQueries(cache, queries));
585 } else {
586 doc.put(HL_CACHE, cache);
587 }
588 } else {
589 doc.put(fessConfig.getIndexFieldCache(), StringUtil.EMPTY);
590 doc.put(HL_CACHE, StringUtil.EMPTY);
591 }
592
593 try {
594 final Template template = handlebars.compile(cacheTemplateName);
595 final Context hbsContext = Context.newContext(doc);
596 return template.apply(hbsContext);
597 } catch (final Exception e) {
598 logger.warn("Failed to create a cache response.", e);
599 }
600
601 return null;
602 }
603
604 protected String replaceHighlightQueries(final String cache, final String[] queries) {
605 final StringBuffer buf = new StringBuffer(cache.length() + 100);
606 final StringBuffer segBuf = new StringBuffer(1000);
607 final Pattern p = Pattern.compile("<[^>]+>");
608 final Matcher m = p.matcher(cache);
609 final String[] regexQueries = new String[queries.length];
610 final String[] hlQueries = new String[queries.length];
611 for (int i = 0; i < queries.length; i++) {
612 regexQueries[i] = Pattern.quote(queries[i]);
613 hlQueries[i] = highlightTagPre + queries[i] + highlightTagPost;
614 }
615 while (m.find()) {
616 segBuf.setLength(0);
617 m.appendReplacement(segBuf, StringUtil.EMPTY);
618 String segment = segBuf.toString();
619 for (int i = 0; i < queries.length; i++) {
620 segment = Pattern.compile(regexQueries[i], Pattern.CASE_INSENSITIVE).matcher(segment).replaceAll(hlQueries[i]);
621 }
622 buf.append(segment);
623 buf.append(m.group(0));
624 }
625 segBuf.setLength(0);
626 m.appendTail(segBuf);
627 String segment = segBuf.toString();
628 for (int i = 0; i < queries.length; i++) {
629 segment = Pattern.compile(regexQueries[i], Pattern.CASE_INSENSITIVE).matcher(segment).replaceAll(hlQueries[i]);
630 }
631 buf.append(segment);
632 return buf.toString();
633 }
634
635 public Object getSitePath(final Map<String, Object> docMap) {
636 final FessConfig fessConfig = ComponentUtil.getFessConfig();
637 final Object siteValue = docMap.get(fessConfig.getIndexFieldSite());
638 if (siteValue != null) {
639 final String site = siteValue.toString();
640 final int size = fessConfig.getResponseMaxSitePathLengthAsInteger();
641 if (size > -1) {
642 return StringUtils.abbreviate(site, size);
643 }
644 return site;
645 }
646 final Object urlLink = docMap.get(fessConfig.getResponseFieldUrlLink());
647 if (urlLink != null) {
648 final String returnUrl;
649 final String url = urlLink.toString();
650 if (LOCAL_PATH_PATTERN.matcher(url).find() || SHARED_FOLDER_PATTERN.matcher(url).find()) {
651 returnUrl = url.replaceFirst("^file:/+", "");
652 } else if (url.startsWith("file:")) {
653 returnUrl = url.replaceFirst("^file:/+", "/");
654 } else {
655 returnUrl = url.replaceFirst("^[a-zA-Z0-9]*:/+", "");
656 }
657 final int size = fessConfig.getResponseMaxSitePathLengthAsInteger();
658 if (size > -1) {
659 return StringUtils.abbreviate(returnUrl, size);
660 }
661 return returnUrl;
662 }
663 return null;
664 }
665
666 public StreamResponse asContentResponse(final Map<String, Object> doc) {
667 if (logger.isDebugEnabled()) {
668 logger.debug("writing the content of: {}", doc);
669 }
670 final FessConfig fessConfig = ComponentUtil.getFessConfig();
671 final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
672 final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
673 if (configId == null) {
674 throw new FessSystemException("configId is null.");
675 }
676 if (configId.length() < 2) {
677 throw new FessSystemException("Invalid configId: " + configId);
678 }
679 final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
680 if (config == null) {
681 throw new FessSystemException("No crawlingConfig: " + configId);
682 }
683 final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
684 final CrawlerClientFactory crawlerClientFactory =
685 config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
686 final CrawlerClient client = crawlerClientFactory.getClient(url);
687 if (client == null) {
688 throw new FessSystemException("No CrawlerClient: " + configId + ", url: " + url);
689 }
690 return writeContent(configId, url, client);
691 }
692
693 protected StreamResponse writeContent(final String configId, final String url, final CrawlerClient client) {
694 final StreamResponse response = new StreamResponse(StringUtil.EMPTY);
695 final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
696 if (responseData.getHttpStatusCode() == 404) {
697 response.httpStatus(responseData.getHttpStatusCode());
698 CloseableUtil.closeQuietly(responseData);
699 return response;
700 }
701 writeFileName(response, responseData);
702 writeContentType(response, responseData);
703 writeNoCache(response, responseData);
704 response.stream(out -> {
705 try (final InputStream is = new BufferedInputStream(responseData.getResponseBody())) {
706 out.write(is);
707 } catch (final IOException e) {
708 if (!(e.getCause() instanceof ClientAbortException)) {
709 throw new FessSystemException("Failed to write a content. configId: " + configId + ", url: " + url, e);
710 }
711 } finally {
712 CloseableUtil.closeQuietly(responseData);
713 }
714 if (logger.isDebugEnabled()) {
715 logger.debug("Finished to write {}", url);
716 }
717 });
718 return response;
719 }
720
721 protected void writeNoCache(final StreamResponse response, final ResponseData responseData) {
722 response.header("Pragma", "no-cache");
723 response.header("Cache-Control", "no-cache");
724 response.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT");
725 }
726
727 protected void writeFileName(final StreamResponse response, final ResponseData responseData) {
728 String charset = responseData.getCharSet();
729 if (charset == null) {
730 charset = Constants.UTF_8;
731 }
732 final String name;
733 final String url = responseData.getUrl();
734 final int pos = url.lastIndexOf('/');
735 try {
736 if (pos >= 0 && pos + 1 < url.length()) {
737 name = URLDecoder.decode(url.substring(pos + 1), charset);
738 } else {
739 name = URLDecoder.decode(url, charset);
740 }
741
742 final String contentDispositionType;
743 if (inlineMimeTypeSet.contains(responseData.getMimeType())) {
744 contentDispositionType = "inline";
745 } else {
746 contentDispositionType = "attachment";
747 }
748
749 final String encodedName = URLEncoder.encode(name, Constants.UTF_8).replace("+", "%20");
750 response.header(CONTENT_DISPOSITION, contentDispositionType + "; filename=\"" + name + "\"; filename*=utf-8''" + encodedName);
751 } catch (final Exception e) {
752 logger.warn("Failed to write a filename: {}", responseData, e);
753 }
754 }
755
756 protected void writeContentType(final StreamResponse response, final ResponseData responseData) {
757 final String mimeType = responseData.getMimeType();
758 if (logger.isDebugEnabled()) {
759 logger.debug("mimeType: {}", mimeType);
760 }
761 if (mimeType == null) {
762 response.contentTypeOctetStream();
763 return;
764 }
765 if (mimeType.startsWith("text/")) {
766 final String charset = LaResponseUtil.getResponse().getCharacterEncoding();
767 if (charset != null) {
768 response.contentType(mimeType + "; charset=" + charset);
769 return;
770 }
771 }
772 response.contentType(mimeType);
773 }
774
775 public String getClientIp(final HttpServletRequest request) {
776 final String value = request.getHeader("x-forwarded-for");
777 if (StringUtil.isNotBlank(value)) {
778 return value;
779 }
780 return request.getRemoteAddr();
781 }
782
783 public FacetResponse getCachedFacetResponse(final String query) {
784 final OptionalThing<FessUserBean> userBean = ComponentUtil.getComponent(FessLoginAssist.class).getSavedUserBean();
785 final String permissionKey = userBean.map(user -> StreamUtil.stream(user.getPermissions())
786 .get(stream -> stream.sorted().distinct().collect(Collectors.joining("\n")))).orElse(StringUtil.EMPTY);
787
788 try {
789 return facetCache.get(query + "\n" + permissionKey, () -> {
790 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
791 final SearchForm params = new SearchForm() {
792 @Override
793 public int getPageSize() {
794 return 0;
795 }
796
797 @Override
798 public int getStartPosition() {
799 return 0;
800 }
801 };
802 params.q = query;
803 final SearchRenderData data = new SearchRenderData();
804 searchHelper.search(params, data, userBean);
805 if (logger.isDebugEnabled()) {
806 logger.debug("loaded facet data: {}", data);
807 }
808 return data.getFacetResponse();
809 });
810 } catch (final ExecutionException e) {
811 throw new FessSystemException("Cannot load facet from cache.", e);
812 }
813 }
814
815 public String createHighlightText(final HighlightField highlightField) {
816 final Text[] fragments = highlightField.fragments();
817 if (fragments != null && fragments.length != 0) {
818 final String[] texts = new String[fragments.length];
819 for (int i = 0; i < fragments.length; i++) {
820 texts[i] = fragments[i].string();
821 }
822 String value = StringUtils.join(texts, ELLIPSIS);
823 if (StringUtil.isNotBlank(value) && !ComponentUtil.getFessConfig().endsWithFullstop(value)) {
824 return value + ELLIPSIS;
825 }
826 return value;
827 }
828 return null;
829 }
830
831 public TextFragment[] createTextFragmentsByHighlight(final HighlightField[] fields) {
832 final List<TextFragment> list = new ArrayList<>();
833 for (final HighlightField field : fields) {
834 final Text[] fragments = field.fragments();
835 if (fragments != null) {
836 for (final Text fragment : fragments) {
837 final String text = fragment.string();
838 if (text.length() > textFragmentPrefixLength + textFragmentSuffixLength) {
839 final String target =
840 text.replace(originalHighlightTagPre, StringUtil.EMPTY).replace(originalHighlightTagPost, StringUtil.EMPTY);
841 if (target.length() > textFragmentPrefixLength + textFragmentSuffixLength) {
842 list.add(new TextFragment(null, target.substring(0, textFragmentPrefixLength),
843 target.substring(target.length() - textFragmentSuffixLength), null));
844 }
845 }
846 }
847 }
848 }
849 return list.toArray(n -> new TextFragment[n]);
850 }
851
852 public TextFragment[] createTextFragmentsByQuery() {
853 return LaRequestUtil.getOptionalRequest().map(req -> {
854 @SuppressWarnings("unchecked")
855 Set<String> querySet = (Set<String>) req.getAttribute(Constants.HIGHLIGHT_QUERIES);
856 if (querySet != null) {
857 return querySet.stream().map(s -> new TextFragment(null, s, null, null)).toArray(n -> new TextFragment[n]);
858 }
859 return new TextFragment[0];
860 }).orElse(new TextFragment[0]);
861 }
862
863 public boolean isUseSession() {
864 return useSession;
865 }
866
867 public void setUseSession(final boolean useSession) {
868 this.useSession = useSession;
869 }
870
871 public void addInitFacetParam(final String key, final String value) {
872 initFacetParamMap.put(value, key);
873 }
874
875 public Map<String, String> getInitFacetParamMap() {
876 return initFacetParamMap;
877 }
878
879 public void addInitGeoParam(final String key, final String value) {
880 initGeoParamMap.put(value, key);
881 }
882
883 public Map<String, String> getInitGeoParamMap() {
884 return initGeoParamMap;
885 }
886
887 public void addFacetQueryView(final FacetQueryView facetQueryView) {
888 facetQueryViewList.add(facetQueryView);
889 }
890
891 public List<FacetQueryView> getFacetQueryViewList() {
892 return facetQueryViewList;
893 }
894
895 public void addInlineMimeType(final String mimeType) {
896 inlineMimeTypeSet.add(mimeType);
897 }
898
899 public ActionHook getActionHook() {
900 return actionHook;
901 }
902
903 public void setActionHook(final ActionHook actionHook) {
904 this.actionHook = actionHook;
905 }
906
907 public void setEncodeUrlLink(final boolean encodeUrlLink) {
908 this.encodeUrlLink = encodeUrlLink;
909 }
910
911 public void setUrlLinkEncoding(final String urlLinkEncoding) {
912 this.urlLinkEncoding = urlLinkEncoding;
913 }
914
915 public void setOriginalHighlightTagPre(final String originalHighlightTagPre) {
916 this.originalHighlightTagPre = originalHighlightTagPre;
917 }
918
919 public void setOriginalHighlightTagPost(final String originalHighlightTagPost) {
920 this.originalHighlightTagPost = originalHighlightTagPost;
921 }
922
923 public void setCacheTemplateName(final String cacheTemplateName) {
924 this.cacheTemplateName = cacheTemplateName;
925 }
926
927 public void setFacetCacheDuration(final long facetCacheDuration) {
928 this.facetCacheDuration = facetCacheDuration;
929 }
930
931 public static class ActionHook {
932
933 public ActionResponse godHandPrologue(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
934 return func.apply(runtime);
935 }
936
937 public ActionResponse godHandMonologue(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
938 return func.apply(runtime);
939 }
940
941 public void godHandEpilogue(final ActionRuntime runtime, final Consumer<ActionRuntime> consumer) {
942 consumer.accept(runtime);
943 }
944
945 public ActionResponse hookBefore(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
946 return func.apply(runtime);
947 }
948
949 public void hookFinally(final ActionRuntime runtime, final Consumer<ActionRuntime> consumer) {
950 consumer.accept(runtime);
951 }
952 }
953
954
955 public static class TextFragment {
956 private String prefix;
957 private String textStart;
958 private String textEnd;
959 private String suffix;
960
961 TextFragment(final String prefix, final String textStart, final String textEnd, final String suffix) {
962 this.prefix = prefix;
963 this.textStart = textStart == null ? StringUtil.EMPTY : textStart;
964 this.textEnd = textEnd;
965 this.suffix = suffix;
966 }
967
968 public String toURLString() {
969 final StringBuilder buf = new StringBuilder();
970 buf.append("text=");
971 if (StringUtil.isNotBlank(prefix)) {
972 buf.append(encodeToString(prefix)).append("-,");
973 }
974 buf.append(encodeToString(textStart));
975 if (StringUtil.isNotBlank(textEnd)) {
976 buf.append(',').append(encodeToString(textEnd));
977 }
978 if (StringUtil.isNotBlank(suffix)) {
979 buf.append(",-").append(encodeToString(suffix));
980 }
981 return buf.toString();
982 }
983
984 private String encodeToString(final String text) {
985 return URLEncoder.encode(text, Constants.CHARSET_UTF_8);
986 }
987 }
988 }