View Javadoc
1   /*
2    * Copyright 2012-2020 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.helper;
17  
18  import static org.codelibs.core.stream.StreamUtil.split;
19  
20  import java.io.BufferedInputStream;
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.InputStream;
24  import java.io.UnsupportedEncodingException;
25  import java.net.URLDecoder;
26  import java.net.URLEncoder;
27  import java.util.ArrayList;
28  import java.util.Date;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.List;
32  import java.util.Locale;
33  import java.util.Map;
34  import java.util.Set;
35  import java.util.concurrent.ConcurrentHashMap;
36  import java.util.concurrent.ExecutionException;
37  import java.util.concurrent.TimeUnit;
38  import java.util.function.Consumer;
39  import java.util.function.Function;
40  import java.util.regex.Matcher;
41  import java.util.regex.Pattern;
42  import java.util.stream.Collectors;
43  
44  import javax.annotation.PostConstruct;
45  import javax.servlet.ServletContext;
46  import javax.servlet.SessionTrackingMode;
47  import javax.servlet.http.HttpServletRequest;
48  import javax.servlet.http.HttpSession;
49  
50  import org.apache.catalina.connector.ClientAbortException;
51  import org.apache.commons.lang3.StringUtils;
52  import org.apache.commons.text.StringEscapeUtils;
53  import org.apache.logging.log4j.LogManager;
54  import org.apache.logging.log4j.Logger;
55  import org.codelibs.core.CoreLibConstants;
56  import org.codelibs.core.io.CloseableUtil;
57  import org.codelibs.core.lang.StringUtil;
58  import org.codelibs.core.misc.DynamicProperties;
59  import org.codelibs.core.stream.StreamUtil;
60  import org.codelibs.fess.Constants;
61  import org.codelibs.fess.app.web.base.SearchForm;
62  import org.codelibs.fess.app.web.base.login.FessLoginAssist;
63  import org.codelibs.fess.crawler.builder.RequestDataBuilder;
64  import org.codelibs.fess.crawler.client.CrawlerClient;
65  import org.codelibs.fess.crawler.client.CrawlerClientFactory;
66  import org.codelibs.fess.crawler.entity.ResponseData;
67  import org.codelibs.fess.crawler.util.CharUtil;
68  import org.codelibs.fess.entity.FacetQueryView;
69  import org.codelibs.fess.entity.HighlightInfo;
70  import org.codelibs.fess.entity.SearchRenderData;
71  import org.codelibs.fess.es.config.exentity.CrawlingConfig;
72  import org.codelibs.fess.exception.FessSystemException;
73  import org.codelibs.fess.helper.UserAgentHelper.UserAgentType;
74  import org.codelibs.fess.mylasta.action.FessUserBean;
75  import org.codelibs.fess.mylasta.direction.FessConfig;
76  import org.codelibs.fess.util.ComponentUtil;
77  import org.codelibs.fess.util.DocumentUtil;
78  import org.codelibs.fess.util.FacetResponse;
79  import org.codelibs.fess.util.ResourceUtil;
80  import org.dbflute.optional.OptionalThing;
81  import org.lastaflute.taglib.function.LaFunctions;
82  import org.lastaflute.web.response.ActionResponse;
83  import org.lastaflute.web.response.StreamResponse;
84  import org.lastaflute.web.ruts.process.ActionRuntime;
85  import org.lastaflute.web.util.LaRequestUtil;
86  import org.lastaflute.web.util.LaResponseUtil;
87  import org.lastaflute.web.util.LaServletContextUtil;
88  
89  import com.github.jknack.handlebars.Context;
90  import com.github.jknack.handlebars.Handlebars;
91  import com.github.jknack.handlebars.Template;
92  import com.github.jknack.handlebars.io.FileTemplateLoader;
93  import com.google.common.cache.Cache;
94  import com.google.common.cache.CacheBuilder;
95  import com.ibm.icu.text.SimpleDateFormat;
96  
97  public class ViewHelper {
98  
99      private static final Logger logger = LogManager.getLogger(ViewHelper.class);
100 
101     protected static final String SCREEN_WIDTH = "screen_width";
102 
103     protected static final int TABLET_WIDTH = 768;
104 
105     protected static final String CONTENT_DISPOSITION = "Content-Disposition";
106 
107     protected static final String HL_CACHE = "hl_cache";
108 
109     protected static final String QUERIES = "queries";
110 
111     protected static final String CACHE_MSG = "cache_msg";
112 
113     protected static final Pattern LOCAL_PATH_PATTERN = Pattern.compile("^file:/+[a-zA-Z]:");
114 
115     protected static final Pattern SHARED_FOLDER_PATTERN = Pattern.compile("^file:/+[^/]\\.");
116 
117     protected boolean encodeUrlLink = false;
118 
119     protected String urlLinkEncoding = Constants.UTF_8;
120 
121     protected String[] highlightedFields;
122 
123     protected String originalHighlightTagPre = "<em>";
124 
125     protected String originalHighlightTagPost = "</em>";
126 
127     protected String highlightTagPre;
128 
129     protected String highlightTagPost;
130 
131     protected boolean useSession = true;
132 
133     protected final Map<String, String> pageCacheMap = new ConcurrentHashMap<>();
134 
135     protected final Map<String, String> initFacetParamMap = new HashMap<>();
136 
137     protected final Map<String, String> initGeoParamMap = new HashMap<>();
138 
139     protected final List<FacetQueryView> facetQueryViewList = new ArrayList<>();
140 
141     protected String cacheTemplateName = "cache";
142 
143     protected String escapedHighlightPre = null;
144 
145     protected String escapedHighlightPost = null;
146 
147     protected Set<Integer> highlightTerminalCharSet = new HashSet<>();
148 
149     protected ActionHook actionHook = new ActionHook();
150 
151     protected final Set<String> inlineMimeTypeSet = new HashSet<>();
152 
153     protected Cache<String, FacetResponse> facetCache;
154 
155     protected long facetCacheDuration = 60 * 10L; // 10min
156 
157     @PostConstruct
158     public void init() {
159         if (logger.isDebugEnabled()) {
160             logger.debug("Initialize {}", this.getClass().getSimpleName());
161         }
162         final FessConfig fessConfig = ComponentUtil.getFessConfig();
163         escapedHighlightPre = LaFunctions.h(originalHighlightTagPre);
164         escapedHighlightPost = LaFunctions.h(originalHighlightTagPost);
165         highlightTagPre = fessConfig.getQueryHighlightTagPre();
166         highlightTagPost = fessConfig.getQueryHighlightTagPost();
167         highlightedFields = fessConfig.getQueryHighlightContentDescriptionFieldsAsArray();
168         for (final int v : fessConfig.getQueryHighlightTerminalCharsAsArray()) {
169             highlightTerminalCharSet.add(v);
170         }
171         try {
172             final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
173             servletContext.setSessionTrackingModes(fessConfig.getSessionTrackingModesAsSet().stream().map(SessionTrackingMode::valueOf)
174                     .collect(Collectors.toSet()));
175         } catch (final Throwable t) {
176             logger.warn("Failed to set SessionTrackingMode.", t);
177         }
178 
179         split(fessConfig.getQueryFacetQueries(), "\n").of(stream -> stream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(s -> {
180             final String[] values = StringUtils.split(s, ":", 2);
181             if (values.length != 2) {
182                 return;
183             }
184             final FacetQueryView#FacetQueryView">FacetQueryView facetQueryView = new FacetQueryView();
185             facetQueryView.setTitle(values[0]);
186             split(values[1], "\t").of(subStream -> subStream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(v -> {
187                 final String[] facet = StringUtils.split(v, "=", 2);
188                 if (facet.length == 2) {
189                     facetQueryView.addQuery(facet[0], facet[1]);
190                 }
191             }));
192             facetQueryView.init();
193             facetQueryViewList.add(facetQueryView);
194             if (logger.isDebugEnabled()) {
195                 logger.debug("loaded {}", facetQueryView);
196             }
197         }));
198 
199         facetCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(facetCacheDuration, TimeUnit.SECONDS).build();
200     }
201 
202     public String getContentTitle(final Map<String, Object> document) {
203         final FessConfig fessConfig = ComponentUtil.getFessConfig();
204         String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
205         if (StringUtil.isBlank(title)) {
206             title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
207             if (StringUtil.isBlank(title)) {
208                 title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
209             }
210         }
211         final int size = fessConfig.getResponseMaxTitleLengthAsInteger();
212         if (size > -1) {
213             title = StringUtils.abbreviate(title, size);
214         }
215         final String value = LaFunctions.h(title);
216         if (!fessConfig.isResponseHighlightContentTitleEnabled()) {
217             return value;
218         }
219         return getQuerySet().map(querySet -> {
220             final String pattern = querySet.stream().map(LaFunctions::h).map(Pattern::quote).collect(Collectors.joining("|"));
221             if (StringUtil.isBlank(pattern)) {
222                 return null;
223             }
224             final Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE).matcher(value);
225             final StringBuffer buf = new StringBuffer(value.length() + 100);
226             while (matcher.find()) {
227                 matcher.appendReplacement(buf, highlightTagPre + matcher.group(0) + highlightTagPost);
228             }
229             matcher.appendTail(buf);
230             return buf.toString();
231         }).orElse(value);
232     }
233 
234     protected OptionalThing<Set<String>> getQuerySet() {
235         return LaRequestUtil.getOptionalRequest().map(req -> {
236             @SuppressWarnings("unchecked")
237             final Set<String> querySet = (Set<String>) req.getAttribute(Constants.HIGHLIGHT_QUERIES);
238             return querySet;
239         }).filter(s -> s != null);
240     }
241 
242     public String getContentDescription(final Map<String, Object> document) {
243         for (final String field : highlightedFields) {
244             final String text = DocumentUtil.getValue(document, field, String.class);
245             if (StringUtil.isNotBlank(text)) {
246                 return escapeHighlight(text);
247             }
248         }
249 
250         return StringUtil.EMPTY;
251     }
252 
253     protected String escapeHighlight(final String text) {
254         final String escaped = LaFunctions.h(text);
255         final String value;
256         if (ComponentUtil.getFessConfig().isQueryHighlightBoundaryPositionDetect()) {
257             int pos = escaped.indexOf(escapedHighlightPre);
258             while (pos >= 0) {
259                 final int c = escaped.codePointAt(pos);
260                 if (Character.isISOControl(c) || highlightTerminalCharSet.contains(c)) {
261                     break;
262                 }
263                 pos--;
264             }
265 
266             value = escaped.substring(pos + 1);
267         } else {
268             value = escaped;
269         }
270         return value.replaceAll(escapedHighlightPre, highlightTagPre).replaceAll(escapedHighlightPost, highlightTagPost);
271     }
272 
273     protected String removeHighlightTag(final String str) {
274         return str.replaceAll(originalHighlightTagPre, StringUtil.EMPTY).replaceAll(originalHighlightTagPost, StringUtil.EMPTY);
275     }
276 
277     public HighlightInfo createHighlightInfo() {
278         return LaRequestUtil.getOptionalRequest().map(req -> {
279             final HighlightInfol#HighlightInfo">HighlightInfo highlightInfo = new HighlightInfo();
280             final String widthStr = req.getParameter(SCREEN_WIDTH);
281             if (StringUtil.isNotBlank(widthStr)) {
282                 final int width = Integer.parseInt(widthStr);
283                 updateHighlisthInfo(highlightInfo, width);
284                 final HttpSession session = req.getSession(false);
285                 if (session != null) {
286                     session.setAttribute(SCREEN_WIDTH, width);
287                 }
288             } else {
289                 final HttpSession session = req.getSession(false);
290                 if (session != null) {
291                     final Integer width = (Integer) session.getAttribute(SCREEN_WIDTH);
292                     if (width != null) {
293                         updateHighlisthInfo(highlightInfo, width);
294                     }
295                 }
296             }
297             return highlightInfo;
298         }).orElse(new HighlightInfo());
299     }
300 
301     protected void updateHighlisthInfo(final HighlightInfo highlightInfo, final int width) {
302         if (width < TABLET_WIDTH) {
303             float ratio = ((float) width) / ((float) TABLET_WIDTH);
304             if (ratio < 0.5) {
305                 ratio = 0.5f;
306             }
307             highlightInfo.fragmentSize((int) (highlightInfo.getFragmentSize() * ratio));
308         }
309     }
310 
311     public String getUrlLink(final Map<String, Object> document) {
312         final FessConfig fessConfig = ComponentUtil.getFessConfig();
313         String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
314 
315         if (StringUtil.isBlank(url)) {
316             return "#not-found-" + DocumentUtil.getValue(document, fessConfig.getIndexFieldDocId(), String.class);
317         }
318 
319         final boolean isSmbUrl = url.startsWith("smb:") || url.startsWith("smb1:");
320         final boolean isFtpUrl = url.startsWith("ftp:");
321         final boolean isSmbOrFtpUrl = isSmbUrl || isFtpUrl;
322 
323         // replacing url with mapping data
324         url = ComponentUtil.getPathMappingHelper().replaceUrl(url);
325 
326         final boolean isHttpUrl = url.startsWith("http:") || url.startsWith("https:");
327 
328         if (isSmbUrl) {
329             url = url.replace("smb:", "file:");
330             url = url.replace("smb1:", "file:");
331         }
332 
333         if (isHttpUrl && isSmbOrFtpUrl) {
334             //  smb/ftp->http
335             // encode
336             final StringBuilder buf = new StringBuilder(url.length() + 100);
337             for (final char c : url.toCharArray()) {
338                 if (CharUtil.isUrlChar(c)) {
339                     buf.append(c);
340                 } else {
341                     try {
342                         buf.append(URLEncoder.encode(String.valueOf(c), urlLinkEncoding));
343                     } catch (final UnsupportedEncodingException e) {
344                         buf.append(c);
345                     }
346                 }
347             }
348             url = buf.toString();
349         } else if (url.startsWith("file:")) {
350             // file, smb/ftp->http
351             url = updateFileProtocol(url);
352 
353             if (encodeUrlLink) {
354                 return appendQueryParameter(document, url);
355             }
356 
357             // decode
358             if (!isSmbOrFtpUrl) {
359                 // file
360                 try {
361                     url = URLDecoder.decode(url.replace("+", "%2B"), urlLinkEncoding);
362                 } catch (final Exception e) {
363                     if (logger.isDebugEnabled()) {
364                         logger.warn("Failed to decode " + url, e);
365                     }
366                 }
367             }
368         }
369         // http, ftp
370         // nothing
371 
372         return appendQueryParameter(document, url);
373     }
374 
375     protected String updateFileProtocol(String url) {
376         final int pos = url.indexOf(':', 5);
377         final boolean isLocalFile = pos > 0 && pos < 12;
378 
379         final UserAgentType ua = ComponentUtil.getUserAgentHelper().getUserAgentType();
380         final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
381         switch (ua) {
382         case IE:
383             if (isLocalFile) {
384                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.ie", "file://"));
385             } else {
386                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.ie", "file://"));
387             }
388             break;
389         case FIREFOX:
390             if (isLocalFile) {
391                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.firefox", "file://"));
392             } else {
393                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.firefox", "file://///"));
394             }
395             break;
396         case CHROME:
397             if (isLocalFile) {
398                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.chrome", "file://"));
399             } else {
400                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.chrome", "file://"));
401             }
402             break;
403         case SAFARI:
404             if (isLocalFile) {
405                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.safari", "file://"));
406             } else {
407                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.safari", "file:////"));
408             }
409             break;
410         case OPERA:
411             if (isLocalFile) {
412                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.opera", "file://"));
413             } else {
414                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.opera", "file://"));
415             }
416             break;
417         default:
418             if (isLocalFile) {
419                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.other", "file://"));
420             } else {
421                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.other", "file://"));
422             }
423             break;
424         }
425         return url;
426     }
427 
428     protected String appendQueryParameter(final Map<String, Object> document, final String url) {
429         final FessConfig fessConfig = ComponentUtil.getFessConfig();
430         if (fessConfig.isAppendQueryParameter()) {
431             if (url.indexOf('#') >= 0) {
432                 return url;
433             }
434 
435             final String mimetype = DocumentUtil.getValue(document, fessConfig.getIndexFieldMimetype(), String.class);
436             if (StringUtil.isNotBlank(mimetype)) {
437                 if ("application/pdf".equals(mimetype)) {
438                     return appendPDFSearchWord(url);
439                 } else {
440                     // TODO others..
441                     return url;
442                 }
443             }
444         }
445         return url;
446     }
447 
448     protected String appendPDFSearchWord(final String url) {
449         final String queries = (String) LaRequestUtil.getRequest().getAttribute(Constants.REQUEST_QUERIES);
450         if (queries != null) {
451             try {
452                 final StringBuilder buf = new StringBuilder(url.length() + 100);
453                 buf.append(url).append("#search=%22");
454                 buf.append(URLEncoder.encode(queries.trim(), Constants.UTF_8));
455                 buf.append("%22");
456                 return buf.toString();
457             } catch (final UnsupportedEncodingException e) {
458                 logger.warn("Unsupported encoding.", e);
459             }
460         }
461         return url;
462     }
463 
464     public String getPagePath(final String page) {
465         final Locale locale = ComponentUtil.getRequestManager().getUserLocale();
466         final String lang = locale.getLanguage();
467         final String country = locale.getCountry();
468 
469         final String pathLC = getLocalizedPagePath(page, lang, country);
470         final String pLC = pageCacheMap.get(pathLC);
471         if (pLC != null) {
472             return pLC;
473         }
474         if (existsPage(pathLC)) {
475             pageCacheMap.put(pathLC, pathLC);
476             return pathLC;
477         }
478 
479         final String pathL = getLocalizedPagePath(page, lang, null);
480         final String pL = pageCacheMap.get(pathL);
481         if (pL != null) {
482             return pL;
483         }
484         if (existsPage(pathL)) {
485             pageCacheMap.put(pathLC, pathL);
486             return pathL;
487         }
488 
489         final String path = getLocalizedPagePath(page, null, null);
490         final String p = pageCacheMap.get(path);
491         if (p != null) {
492             return p;
493         }
494         if (existsPage(path)) {
495             pageCacheMap.put(pathLC, path);
496             return path;
497         }
498 
499         return "index.jsp";
500     }
501 
502     private String getLocalizedPagePath(final String page, final String lang, final String country) {
503         final StringBuilder buf = new StringBuilder(100);
504         buf.append("/WEB-INF/view/").append(page);
505         if (StringUtil.isNotBlank(lang)) {
506             buf.append('_').append(lang);
507             if (StringUtil.isNotBlank(country)) {
508                 buf.append('_').append(country);
509             }
510         }
511         buf.append(".jsp");
512         return buf.toString();
513     }
514 
515     private boolean existsPage(final String path) {
516         final String realPath = LaServletContextUtil.getServletContext().getRealPath(path);
517         final File file = new File(realPath);
518         return file.isFile();
519     }
520 
521     public String createCacheContent(final Map<String, Object> doc, final String[] queries) {
522         final FessConfig fessConfig = ComponentUtil.getFessConfig();
523         final FileTemplateLoader loader = new FileTemplateLoader(ResourceUtil.getViewTemplatePath().toFile());
524         final Handlebars handlebars = new Handlebars(loader);
525 
526         Locale locale = ComponentUtil.getRequestManager().getUserLocale();
527         if (locale == null) {
528             locale = Locale.ENGLISH;
529         }
530         String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
531         if (url == null) {
532             url = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
533         }
534         doc.put(fessConfig.getResponseFieldUrlLink(), getUrlLink(doc));
535         String createdStr;
536         final Date created = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCreated(), Date.class);
537         if (created != null) {
538             final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
539             createdStr = sdf.format(created);
540         } else {
541             createdStr = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
542         }
543         doc.put(CACHE_MSG, ComponentUtil.getMessageManager()
544                 .getMessage(locale, "labels.search_cache_msg", new Object[] { url, createdStr }));
545 
546         doc.put(QUERIES, queries);
547 
548         String cache = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCache(), String.class);
549         if (cache != null) {
550             final String mimetype = DocumentUtil.getValue(doc, fessConfig.getIndexFieldMimetype(), String.class);
551             if (!ComponentUtil.getFessConfig().isHtmlMimetypeForCache(mimetype)) {
552                 cache = StringEscapeUtils.escapeHtml4(cache);
553             }
554             cache = ComponentUtil.getPathMappingHelper().replaceUrls(cache);
555             if (queries != null && queries.length > 0) {
556                 doc.put(HL_CACHE, replaceHighlightQueries(cache, queries));
557             } else {
558                 doc.put(HL_CACHE, cache);
559             }
560         } else {
561             doc.put(fessConfig.getIndexFieldCache(), StringUtil.EMPTY);
562             doc.put(HL_CACHE, StringUtil.EMPTY);
563         }
564 
565         try {
566             final Template template = handlebars.compile(cacheTemplateName);
567             final Context hbsContext = Context.newContext(doc);
568             return template.apply(hbsContext);
569         } catch (final Exception e) {
570             logger.warn("Failed to create a cache response.", e);
571         }
572 
573         return null;
574     }
575 
576     protected String replaceHighlightQueries(final String cache, final String[] queries) {
577         final StringBuffer buf = new StringBuffer(cache.length() + 100);
578         final StringBuffer segBuf = new StringBuffer(1000);
579         final Pattern p = Pattern.compile("<[^>]+>");
580         final Matcher m = p.matcher(cache);
581         final String[] regexQueries = new String[queries.length];
582         final String[] hlQueries = new String[queries.length];
583         for (int i = 0; i < queries.length; i++) {
584             regexQueries[i] = Pattern.quote(queries[i]);
585             hlQueries[i] = highlightTagPre + queries[i] + highlightTagPost;
586         }
587         while (m.find()) {
588             segBuf.setLength(0);
589             m.appendReplacement(segBuf, StringUtil.EMPTY);
590             String segment = segBuf.toString();
591             for (int i = 0; i < queries.length; i++) {
592                 segment = Pattern.compile(regexQueries[i], Pattern.CASE_INSENSITIVE).matcher(segment).replaceAll(hlQueries[i]);
593             }
594             buf.append(segment);
595             buf.append(m.group(0));
596         }
597         segBuf.setLength(0);
598         m.appendTail(segBuf);
599         String segment = segBuf.toString();
600         for (int i = 0; i < queries.length; i++) {
601             segment = Pattern.compile(regexQueries[i], Pattern.CASE_INSENSITIVE).matcher(segment).replaceAll(hlQueries[i]);
602         }
603         buf.append(segment);
604         return buf.toString();
605     }
606 
607     public Object getSitePath(final Map<String, Object> docMap) {
608         final FessConfig fessConfig = ComponentUtil.getFessConfig();
609         final Object urlLink = docMap.get(fessConfig.getResponseFieldUrlLink());
610         if (urlLink != null) {
611             final String returnUrl;
612             final String url = urlLink.toString();
613             if (LOCAL_PATH_PATTERN.matcher(url).find() || SHARED_FOLDER_PATTERN.matcher(url).find()) {
614                 returnUrl = url.replaceFirst("^file:/+", "");
615             } else if (url.startsWith("file:")) {
616                 returnUrl = url.replaceFirst("^file:/+", "/");
617             } else {
618                 returnUrl = url.replaceFirst("^[a-zA-Z0-9]*:/+", "");
619             }
620             final int size = fessConfig.getResponseMaxSitePathLengthAsInteger();
621             if (size > -1) {
622                 return StringUtils.abbreviate(returnUrl, size);
623             } else {
624                 return returnUrl;
625             }
626         }
627         return null;
628     }
629 
630     public StreamResponse asContentResponse(final Map<String, Object> doc) {
631         if (logger.isDebugEnabled()) {
632             logger.debug("writing the content of: {}", doc);
633         }
634         final FessConfig fessConfig = ComponentUtil.getFessConfig();
635         final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
636         final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
637         if (configId == null) {
638             throw new FessSystemException("configId is null.");
639         }
640         if (configId.length() < 2) {
641             throw new FessSystemException("Invalid configId: " + configId);
642         }
643         final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
644         if (config == null) {
645             throw new FessSystemException("No crawlingConfig: " + configId);
646         }
647         final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
648         final CrawlerClientFactory crawlerClientFactory =
649                 config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
650         final CrawlerClient client = crawlerClientFactory.getClient(url);
651         if (client == null) {
652             throw new FessSystemException("No CrawlerClient: " + configId + ", url: " + url);
653         }
654         return writeContent(configId, url, client);
655     }
656 
657     protected StreamResponse writeContent(final String configId, final String url, final CrawlerClient client) {
658         final StreamResponse response = new StreamResponse(StringUtil.EMPTY);
659         final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
660         if (responseData.getHttpStatusCode() == 404) {
661             response.httpStatus(responseData.getHttpStatusCode());
662             CloseableUtil.closeQuietly(responseData);
663             return response;
664         }
665         writeFileName(response, responseData);
666         writeContentType(response, responseData);
667         writeNoCache(response, responseData);
668         response.stream(out -> {
669             try (final InputStream is = new BufferedInputStream(responseData.getResponseBody())) {
670                 out.write(is);
671             } catch (final IOException e) {
672                 if (!(e.getCause() instanceof ClientAbortException)) {
673                     throw new FessSystemException("Failed to write a content. configId: " + configId + ", url: " + url, e);
674                 }
675             } finally {
676                 CloseableUtil.closeQuietly(responseData);
677             }
678             if (logger.isDebugEnabled()) {
679                 logger.debug("Finished to write {}", url);
680             }
681         });
682         return response;
683     }
684 
685     protected void writeNoCache(final StreamResponse response, final ResponseData responseData) {
686         response.header("Pragma", "no-cache");
687         response.header("Cache-Control", "no-cache");
688         response.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT");
689     }
690 
691     protected void writeFileName(final StreamResponse response, final ResponseData responseData) {
692         String charset = responseData.getCharSet();
693         if (charset == null) {
694             charset = Constants.UTF_8;
695         }
696         final String name;
697         final String url = responseData.getUrl();
698         final int pos = url.lastIndexOf('/');
699         try {
700             if (pos >= 0 && pos + 1 < url.length()) {
701                 name = URLDecoder.decode(url.substring(pos + 1), charset);
702             } else {
703                 name = URLDecoder.decode(url, charset);
704             }
705 
706             final String contentDispositionType;
707             if (inlineMimeTypeSet.contains(responseData.getMimeType())) {
708                 contentDispositionType = "inline";
709             } else {
710                 contentDispositionType = "attachment";
711             }
712 
713             final String encodedName = URLEncoder.encode(name, Constants.UTF_8).replace("+", "%20");
714             response.header(CONTENT_DISPOSITION, contentDispositionType + "; filename=\"" + name + "\"; filename*=utf-8''" + encodedName);
715         } catch (final Exception e) {
716             logger.warn("Failed to write a filename: " + responseData, e);
717         }
718     }
719 
720     protected void writeContentType(final StreamResponse response, final ResponseData responseData) {
721         final String mimeType = responseData.getMimeType();
722         if (logger.isDebugEnabled()) {
723             logger.debug("mimeType: {}", mimeType);
724         }
725         if (mimeType == null) {
726             response.contentTypeOctetStream();
727             return;
728         }
729         if (mimeType.startsWith("text/")) {
730             final String charset = LaResponseUtil.getResponse().getCharacterEncoding();
731             if (charset != null) {
732                 response.contentType(mimeType + "; charset=" + charset);
733                 return;
734             }
735         }
736         response.contentType(mimeType);
737     }
738 
739     public String getClientIp(final HttpServletRequest request) {
740         final String value = request.getHeader("x-forwarded-for");
741         if (StringUtil.isNotBlank(value)) {
742             return value;
743         } else {
744             return request.getRemoteAddr();
745         }
746     }
747 
748     public FacetResponse getCachedFacetResponse(final String query) {
749         final OptionalThing<FessUserBean> userBean = ComponentUtil.getComponent(FessLoginAssist.class).getSavedUserBean();
750         final String permissionKey =
751                 userBean.map(
752                         user -> StreamUtil.stream(user.getPermissions()).get(
753                                 stream -> stream.sorted().distinct().collect(Collectors.joining("\n")))).orElse(StringUtil.EMPTY);
754 
755         try {
756             return facetCache.get(query + "\n" + permissionKey, () -> {
757                 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
758                 final SearchFormearchForm.html#SearchForm">SearchForm params = new SearchForm() {
759                     @Override
760                     public int getPageSize() {
761                         return 0;
762                     }
763 
764                     @Override
765                     public int getStartPosition() {
766                         return 0;
767                     }
768                 };
769                 params.q = query;
770                 final SearchRenderDatarData.html#SearchRenderData">SearchRenderData data = new SearchRenderData();
771                 searchHelper.search(params, data, userBean);
772                 if (logger.isDebugEnabled()) {
773                     logger.debug("loaded facet data: {}", data);
774                 }
775                 return data.getFacetResponse();
776             });
777         } catch (final ExecutionException e) {
778             throw new FessSystemException("Cannot load facet from cache.", e);
779         }
780     }
781 
782     public boolean isUseSession() {
783         return useSession;
784     }
785 
786     public void setUseSession(final boolean useSession) {
787         this.useSession = useSession;
788     }
789 
790     public void addInitFacetParam(final String key, final String value) {
791         initFacetParamMap.put(value, key);
792     }
793 
794     public Map<String, String> getInitFacetParamMap() {
795         return initFacetParamMap;
796     }
797 
798     public void addInitGeoParam(final String key, final String value) {
799         initGeoParamMap.put(value, key);
800     }
801 
802     public Map<String, String> getInitGeoParamMap() {
803         return initGeoParamMap;
804     }
805 
806     public void addFacetQueryView(final FacetQueryView facetQueryView) {
807         facetQueryViewList.add(facetQueryView);
808     }
809 
810     public List<FacetQueryView> getFacetQueryViewList() {
811         return facetQueryViewList;
812     }
813 
814     public void addInlineMimeType(final String mimeType) {
815         inlineMimeTypeSet.add(mimeType);
816     }
817 
818     public ActionHook getActionHook() {
819         return actionHook;
820     }
821 
822     public void setActionHook(final ActionHook actionHook) {
823         this.actionHook = actionHook;
824     }
825 
826     public static class ActionHook {
827 
828         public ActionResponse godHandPrologue(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
829             return func.apply(runtime);
830         }
831 
832         public ActionResponse godHandMonologue(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
833             return func.apply(runtime);
834         }
835 
836         public void godHandEpilogue(final ActionRuntime runtime, final Consumer<ActionRuntime> consumer) {
837             consumer.accept(runtime);
838         }
839 
840         public ActionResponse hookBefore(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
841             return func.apply(runtime);
842         }
843 
844         public void hookFinally(final ActionRuntime runtime, final Consumer<ActionRuntime> consumer) {
845             consumer.accept(runtime);
846         }
847     }
848 
849     public void setEncodeUrlLink(final boolean encodeUrlLink) {
850         this.encodeUrlLink = encodeUrlLink;
851     }
852 
853     public void setUrlLinkEncoding(final String urlLinkEncoding) {
854         this.urlLinkEncoding = urlLinkEncoding;
855     }
856 
857     public void setOriginalHighlightTagPre(final String originalHighlightTagPre) {
858         this.originalHighlightTagPre = originalHighlightTagPre;
859     }
860 
861     public void setOriginalHighlightTagPost(final String originalHighlightTagPost) {
862         this.originalHighlightTagPost = originalHighlightTagPost;
863     }
864 
865     public void setCacheTemplateName(final String cacheTemplateName) {
866         this.cacheTemplateName = cacheTemplateName;
867     }
868 
869     public void setFacetCacheDuration(final long facetCacheDuration) {
870         this.facetCacheDuration = facetCacheDuration;
871     }
872 }