View Javadoc
1   /*
2    * Copyright 2012-2020 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.helper;
17  
18  import static org.codelibs.core.stream.StreamUtil.split;
19  
20  import java.io.BufferedInputStream;
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.InputStream;
24  import java.io.UnsupportedEncodingException;
25  import java.net.URLDecoder;
26  import java.net.URLEncoder;
27  import java.util.ArrayList;
28  import java.util.Date;
29  import java.util.HashMap;
30  import java.util.HashSet;
31  import java.util.List;
32  import java.util.Locale;
33  import java.util.Map;
34  import java.util.Set;
35  import java.util.concurrent.ConcurrentHashMap;
36  import java.util.concurrent.ExecutionException;
37  import java.util.concurrent.TimeUnit;
38  import java.util.function.Consumer;
39  import java.util.function.Function;
40  import java.util.regex.Matcher;
41  import java.util.regex.Pattern;
42  import java.util.stream.Collectors;
43  
44  import javax.annotation.PostConstruct;
45  import javax.servlet.ServletContext;
46  import javax.servlet.SessionTrackingMode;
47  import javax.servlet.http.HttpServletRequest;
48  import javax.servlet.http.HttpSession;
49  
50  import org.apache.catalina.connector.ClientAbortException;
51  import org.apache.commons.lang3.StringUtils;
52  import org.apache.commons.text.StringEscapeUtils;
53  import org.apache.logging.log4j.LogManager;
54  import org.apache.logging.log4j.Logger;
55  import org.codelibs.core.CoreLibConstants;
56  import org.codelibs.core.io.CloseableUtil;
57  import org.codelibs.core.lang.StringUtil;
58  import org.codelibs.core.misc.DynamicProperties;
59  import org.codelibs.core.stream.StreamUtil;
60  import org.codelibs.fess.Constants;
61  import org.codelibs.fess.app.web.base.SearchForm;
62  import org.codelibs.fess.app.web.base.login.FessLoginAssist;
63  import org.codelibs.fess.crawler.builder.RequestDataBuilder;
64  import org.codelibs.fess.crawler.client.CrawlerClient;
65  import org.codelibs.fess.crawler.client.CrawlerClientFactory;
66  import org.codelibs.fess.crawler.entity.ResponseData;
67  import org.codelibs.fess.crawler.util.CharUtil;
68  import org.codelibs.fess.entity.FacetQueryView;
69  import org.codelibs.fess.entity.HighlightInfo;
70  import org.codelibs.fess.entity.SearchRenderData;
71  import org.codelibs.fess.es.config.exentity.CrawlingConfig;
72  import org.codelibs.fess.exception.FessSystemException;
73  import org.codelibs.fess.helper.UserAgentHelper.UserAgentType;
74  import org.codelibs.fess.mylasta.action.FessUserBean;
75  import org.codelibs.fess.mylasta.direction.FessConfig;
76  import org.codelibs.fess.util.ComponentUtil;
77  import org.codelibs.fess.util.DocumentUtil;
78  import org.codelibs.fess.util.FacetResponse;
79  import org.codelibs.fess.util.ResourceUtil;
80  import org.dbflute.optional.OptionalThing;
81  import org.lastaflute.taglib.function.LaFunctions;
82  import org.lastaflute.web.response.ActionResponse;
83  import org.lastaflute.web.response.StreamResponse;
84  import org.lastaflute.web.ruts.process.ActionRuntime;
85  import org.lastaflute.web.util.LaRequestUtil;
86  import org.lastaflute.web.util.LaResponseUtil;
87  import org.lastaflute.web.util.LaServletContextUtil;
88  
89  import com.github.jknack.handlebars.Context;
90  import com.github.jknack.handlebars.Handlebars;
91  import com.github.jknack.handlebars.Template;
92  import com.github.jknack.handlebars.io.FileTemplateLoader;
93  import com.google.common.cache.Cache;
94  import com.google.common.cache.CacheBuilder;
95  import com.ibm.icu.text.SimpleDateFormat;
96  
97  public class ViewHelper {
98  
99      private static final Logger logger = LogManager.getLogger(ViewHelper.class);
100 
101     protected static final String SCREEN_WIDTH = "screen_width";
102 
103     protected static final int TABLET_WIDTH = 768;
104 
105     protected static final String CONTENT_DISPOSITION = "Content-Disposition";
106 
107     protected static final String HL_CACHE = "hl_cache";
108 
109     protected static final String QUERIES = "queries";
110 
111     protected static final String CACHE_MSG = "cache_msg";
112 
113     protected static final Pattern LOCAL_PATH_PATTERN = Pattern.compile("^file:/+[a-zA-Z]:");
114 
115     protected static final Pattern SHARED_FOLDER_PATTERN = Pattern.compile("^file:/+[^/]\\.");
116 
117     protected boolean encodeUrlLink = false;
118 
119     protected String urlLinkEncoding = Constants.UTF_8;
120 
121     protected String[] highlightedFields;
122 
123     protected String originalHighlightTagPre = "<em>";
124 
125     protected String originalHighlightTagPost = "</em>";
126 
127     protected String highlightTagPre;
128 
129     protected String highlightTagPost;
130 
131     protected boolean useSession = true;
132 
133     protected final Map<String, String> pageCacheMap = new ConcurrentHashMap<>();
134 
135     protected final Map<String, String> initFacetParamMap = new HashMap<>();
136 
137     protected final Map<String, String> initGeoParamMap = new HashMap<>();
138 
139     protected final List<FacetQueryView> facetQueryViewList = new ArrayList<>();
140 
141     protected String cacheTemplateName = "cache";
142 
143     protected String escapedHighlightPre = null;
144 
145     protected String escapedHighlightPost = null;
146 
147     protected Set<Integer> highlightTerminalCharSet = new HashSet<>();
148 
149     protected ActionHook actionHook = new ActionHook();
150 
151     protected final Set<String> inlineMimeTypeSet = new HashSet<>();
152 
153     protected Cache<String, FacetResponse> facetCache;
154 
155     protected long facetCacheDuration = 60 * 10L; // 10min
156 
157     @PostConstruct
158     public void init() {
159         if (logger.isDebugEnabled()) {
160             logger.debug("Initialize {}", this.getClass().getSimpleName());
161         }
162         final FessConfig fessConfig = ComponentUtil.getFessConfig();
163         escapedHighlightPre = LaFunctions.h(originalHighlightTagPre);
164         escapedHighlightPost = LaFunctions.h(originalHighlightTagPost);
165         highlightTagPre = fessConfig.getQueryHighlightTagPre();
166         highlightTagPost = fessConfig.getQueryHighlightTagPost();
167         highlightedFields = fessConfig.getQueryHighlightContentDescriptionFieldsAsArray();
168         for (final int v : fessConfig.getQueryHighlightTerminalCharsAsArray()) {
169             highlightTerminalCharSet.add(v);
170         }
171         try {
172             final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
173             servletContext.setSessionTrackingModes(fessConfig.getSessionTrackingModesAsSet().stream().map(SessionTrackingMode::valueOf)
174                     .collect(Collectors.toSet()));
175         } catch (final Throwable t) {
176             logger.warn("Failed to set SessionTrackingMode.", t);
177         }
178 
179         split(fessConfig.getQueryFacetQueries(), "\n").of(stream -> stream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(s -> {
180             final String[] values = StringUtils.split(s, ":", 2);
181             if (values.length != 2) {
182                 return;
183             }
184             final FacetQueryView#FacetQueryView">FacetQueryView facetQueryView = new FacetQueryView();
185             facetQueryView.setTitle(values[0]);
186             split(values[1], "\t").of(subStream -> subStream.map(String::trim).filter(StringUtil::isNotEmpty).forEach(v -> {
187                 final String[] facet = StringUtils.split(v, "=", 2);
188                 if (facet.length == 2) {
189                     facetQueryView.addQuery(facet[0], facet[1]);
190                 }
191             }));
192             facetQueryView.init();
193             facetQueryViewList.add(facetQueryView);
194             if (logger.isDebugEnabled()) {
195                 logger.debug("loaded {}", facetQueryView);
196             }
197         }));
198 
199         facetCache = CacheBuilder.newBuilder().maximumSize(1000).expireAfterWrite(facetCacheDuration, TimeUnit.SECONDS).build();
200     }
201 
202     public String getContentTitle(final Map<String, Object> document) {
203         final FessConfig fessConfig = ComponentUtil.getFessConfig();
204         String title = DocumentUtil.getValue(document, fessConfig.getIndexFieldTitle(), String.class);
205         if (StringUtil.isBlank(title)) {
206             title = DocumentUtil.getValue(document, fessConfig.getIndexFieldFilename(), String.class);
207             if (StringUtil.isBlank(title)) {
208                 title = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
209             }
210         }
211         final int size = fessConfig.getResponseMaxTitleLengthAsInteger();
212         if (size > -1) {
213             title = StringUtils.abbreviate(title, size);
214         }
215         final String value = LaFunctions.h(title);
216         if (!fessConfig.isResponseHighlightContentTitleEnabled()) {
217             return value;
218         }
219         return getQuerySet().map(querySet -> {
220             final String pattern = querySet.stream().map(LaFunctions::h).map(Pattern::quote).collect(Collectors.joining("|"));
221             if (StringUtil.isBlank(pattern)) {
222                 return null;
223             }
224             final Matcher matcher = Pattern.compile(pattern, Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE).matcher(value);
225             final StringBuffer buf = new StringBuffer(value.length() + 100);
226             while (matcher.find()) {
227                 matcher.appendReplacement(buf, highlightTagPre + matcher.group(0) + highlightTagPost);
228             }
229             matcher.appendTail(buf);
230             return buf.toString();
231         }).orElse(value);
232     }
233 
234     protected OptionalThing<Set<String>> getQuerySet() {
235         return LaRequestUtil.getOptionalRequest().map(req -> {
236             @SuppressWarnings("unchecked")
237             final Set<String> querySet = (Set<String>) req.getAttribute(Constants.HIGHLIGHT_QUERIES);
238             return querySet;
239         }).filter(s -> s != null);
240     }
241 
242     public String getContentDescription(final Map<String, Object> document) {
243         for (final String field : highlightedFields) {
244             final String text = DocumentUtil.getValue(document, field, String.class);
245             if (StringUtil.isNotBlank(text)) {
246                 return escapeHighlight(text);
247             }
248         }
249 
250         return StringUtil.EMPTY;
251     }
252 
253     protected String escapeHighlight(final String text) {
254         final String escaped = LaFunctions.h(text);
255         final String value;
256         if (ComponentUtil.getFessConfig().isQueryHighlightBoundaryPositionDetect()) {
257             int pos = escaped.indexOf(escapedHighlightPre);
258             while (pos >= 0) {
259                 final int c = escaped.codePointAt(pos);
260                 if (Character.isISOControl(c) || highlightTerminalCharSet.contains(c)) {
261                     break;
262                 }
263                 pos--;
264             }
265 
266             value = escaped.substring(pos + 1);
267         } else {
268             value = escaped;
269         }
270         return value.replaceAll(escapedHighlightPre, highlightTagPre).replaceAll(escapedHighlightPost, highlightTagPost);
271     }
272 
273     protected String removeHighlightTag(final String str) {
274         return str.replaceAll(originalHighlightTagPre, StringUtil.EMPTY).replaceAll(originalHighlightTagPost, StringUtil.EMPTY);
275     }
276 
277     public HighlightInfo createHighlightInfo() {
278         return LaRequestUtil.getOptionalRequest().map(req -> {
279             final HighlightInfol#HighlightInfo">HighlightInfo highlightInfo = new HighlightInfo();
280             final String widthStr = req.getParameter(SCREEN_WIDTH);
281             if (StringUtil.isNotBlank(widthStr)) {
282                 final int width = Integer.parseInt(widthStr);
283                 updateHighlisthInfo(highlightInfo, width);
284                 final HttpSession session = req.getSession(false);
285                 if (session != null) {
286                     session.setAttribute(SCREEN_WIDTH, width);
287                 }
288             } else {
289                 final HttpSession session = req.getSession(false);
290                 if (session != null) {
291                     final Integer width = (Integer) session.getAttribute(SCREEN_WIDTH);
292                     if (width != null) {
293                         updateHighlisthInfo(highlightInfo, width);
294                     }
295                 }
296             }
297             return highlightInfo;
298         }).orElse(new HighlightInfo());
299     }
300 
301     protected void updateHighlisthInfo(final HighlightInfo highlightInfo, final int width) {
302         if (width < TABLET_WIDTH) {
303             float ratio = ((float) width) / ((float) TABLET_WIDTH);
304             if (ratio < 0.5) {
305                 ratio = 0.5f;
306             }
307             highlightInfo.fragmentSize((int) (highlightInfo.getFragmentSize() * ratio));
308         }
309     }
310 
311     public String getUrlLink(final Map<String, Object> document) {
312         final FessConfig fessConfig = ComponentUtil.getFessConfig();
313         String url = DocumentUtil.getValue(document, fessConfig.getIndexFieldUrl(), String.class);
314 
315         if (StringUtil.isBlank(url)) {
316             return "#not-found-" + DocumentUtil.getValue(document, fessConfig.getIndexFieldDocId(), String.class);
317         }
318 
319         final boolean isSmbUrl = url.startsWith("smb:") || url.startsWith("smb1:");
320         final boolean isFtpUrl = url.startsWith("ftp:");
321         final boolean isSmbOrFtpUrl = isSmbUrl || isFtpUrl;
322 
323         // replacing url with mapping data
324         url = ComponentUtil.getPathMappingHelper().replaceUrl(url);
325 
326         final boolean isHttpUrl = url.startsWith("http:") || url.startsWith("https:");
327 
328         if (isSmbUrl) {
329             url = url.replace("smb:", "file:");
330             url = url.replace("smb1:", "file:");
331         }
332 
333         if (isHttpUrl && isSmbOrFtpUrl) {
334             //  smb/ftp->http
335             // encode
336             final StringBuilder buf = new StringBuilder(url.length() + 100);
337             for (final char c : url.toCharArray()) {
338                 if (CharUtil.isUrlChar(c)) {
339                     buf.append(c);
340                 } else {
341                     try {
342                         buf.append(URLEncoder.encode(String.valueOf(c), urlLinkEncoding));
343                     } catch (final UnsupportedEncodingException e) {
344                         buf.append(c);
345                     }
346                 }
347             }
348             url = buf.toString();
349         } else if (url.startsWith("file:")) {
350             // file, smb/ftp->http
351             url = updateFileProtocol(url);
352 
353             if (encodeUrlLink) {
354                 return appendQueryParameter(document, url);
355             }
356 
357             // decode
358             if (!isSmbOrFtpUrl) {
359                 // file
360                 try {
361                     url = URLDecoder.decode(url.replace("+", "%2B"), urlLinkEncoding);
362                 } catch (final Exception e) {
363                     if (logger.isDebugEnabled()) {
364                         logger.warn("Failed to decode " + url, e);
365                     }
366                 }
367             }
368         }
369         // http, ftp
370         // nothing
371 
372         return appendQueryParameter(document, url);
373     }
374 
375     protected String updateFileProtocol(String url) {
376         final int pos = url.indexOf(':', 5);
377         final boolean isLocalFile = pos > 0 && pos < 12;
378 
379         final UserAgentType ua = ComponentUtil.getUserAgentHelper().getUserAgentType();
380         final DynamicProperties systemProperties = ComponentUtil.getSystemProperties();
381         switch (ua) {
382         case IE:
383             if (isLocalFile) {
384                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.ie", "file://"));
385             } else {
386                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.ie", "file://"));
387             }
388             break;
389         case FIREFOX:
390             if (isLocalFile) {
391                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.firefox", "file://"));
392             } else {
393                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.firefox", "file://///"));
394             }
395             break;
396         case CHROME:
397             if (isLocalFile) {
398                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.chrome", "file://"));
399             } else {
400                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.chrome", "file://"));
401             }
402             break;
403         case SAFARI:
404             if (isLocalFile) {
405                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.safari", "file://"));
406             } else {
407                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.safari", "file:////"));
408             }
409             break;
410         case OPERA:
411             if (isLocalFile) {
412                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.opera", "file://"));
413             } else {
414                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.opera", "file://"));
415             }
416             break;
417         default:
418             if (isLocalFile) {
419                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.winlocal.other", "file://"));
420             } else {
421                 url = url.replaceFirst("file:/+", systemProperties.getProperty("file.protocol.other", "file://"));
422             }
423             break;
424         }
425         return url;
426     }
427 
428     protected String appendQueryParameter(final Map<String, Object> document, final String url) {
429         final FessConfig fessConfig = ComponentUtil.getFessConfig();
430         if (fessConfig.isAppendQueryParameter()) {
431             if (url.indexOf('#') >= 0) {
432                 return url;
433             }
434 
435             final String mimetype = DocumentUtil.getValue(document, fessConfig.getIndexFieldMimetype(), String.class);
436             if (StringUtil.isNotBlank(mimetype)) {
437                 if ("application/pdf".equals(mimetype)) {
438                     return appendPDFSearchWord(url);
439                 } else {
440                     // TODO others..
441                     return url;
442                 }
443             }
444         }
445         return url;
446     }
447 
448     protected String appendPDFSearchWord(final String url) {
449         final String queries = (String) LaRequestUtil.getRequest().getAttribute(Constants.REQUEST_QUERIES);
450         if (queries != null) {
451             try {
452                 final StringBuilder buf = new StringBuilder(url.length() + 100);
453                 buf.append(url).append("#search=%22");
454                 buf.append(URLEncoder.encode(queries.trim(), Constants.UTF_8));
455                 buf.append("%22");
456                 return buf.toString();
457             } catch (final UnsupportedEncodingException e) {
458                 logger.warn("Unsupported encoding.", e);
459             }
460         }
461         return url;
462     }
463 
464     public String getPagePath(final String page) {
465         final Locale locale = ComponentUtil.getRequestManager().getUserLocale();
466         final String lang = locale.getLanguage();
467         final String country = locale.getCountry();
468 
469         final String pathLC = getLocalizedPagePath(page, lang, country);
470         final String pLC = pageCacheMap.get(pathLC);
471         if (pLC != null) {
472             return pLC;
473         }
474         if (existsPage(pathLC)) {
475             pageCacheMap.put(pathLC, pathLC);
476             return pathLC;
477         }
478 
479         final String pathL = getLocalizedPagePath(page, lang, null);
480         final String pL = pageCacheMap.get(pathL);
481         if (pL != null) {
482             return pL;
483         }
484         if (existsPage(pathL)) {
485             pageCacheMap.put(pathLC, pathL);
486             return pathL;
487         }
488 
489         final String path = getLocalizedPagePath(page, null, null);
490         final String p = pageCacheMap.get(path);
491         if (p != null) {
492             return p;
493         }
494         if (existsPage(path)) {
495             pageCacheMap.put(pathLC, path);
496             return path;
497         }
498 
499         return "index.jsp";
500     }
501 
502     private String getLocalizedPagePath(final String page, final String lang, final String country) {
503         final StringBuilder buf = new StringBuilder(100);
504         buf.append("/WEB-INF/view/").append(page);
505         if (StringUtil.isNotBlank(lang)) {
506             buf.append('_').append(lang);
507             if (StringUtil.isNotBlank(country)) {
508                 buf.append('_').append(country);
509             }
510         }
511         buf.append(".jsp");
512         return buf.toString();
513     }
514 
515     private boolean existsPage(final String path) {
516         final String realPath = LaServletContextUtil.getServletContext().getRealPath(path);
517         final File file = new File(realPath);
518         return file.isFile();
519     }
520 
521     public String createCacheContent(final Map<String, Object> doc, final String[] queries) {
522         final FessConfig fessConfig = ComponentUtil.getFessConfig();
523         final FileTemplateLoader loader = new FileTemplateLoader(ResourceUtil.getViewTemplatePath().toFile());
524         final Handlebars handlebars = new Handlebars(loader);
525 
526         Locale locale = ComponentUtil.getRequestManager().getUserLocale();
527         if (locale == null) {
528             locale = Locale.ENGLISH;
529         }
530         String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
531         if (url == null) {
532             url = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
533         }
534         doc.put(fessConfig.getResponseFieldUrlLink(), getUrlLink(doc));
535         String createdStr;
536         final Date created = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCreated(), Date.class);
537         if (created != null) {
538             final SimpleDateFormat sdf = new SimpleDateFormat(CoreLibConstants.DATE_FORMAT_ISO_8601_EXTEND);
539             createdStr = sdf.format(created);
540         } else {
541             createdStr = ComponentUtil.getMessageManager().getMessage(locale, "labels.search_unknown");
542         }
543         doc.put(CACHE_MSG, ComponentUtil.getMessageManager().getMessage(locale, "labels.search_cache_msg", url, createdStr));
544 
545         doc.put(QUERIES, queries);
546 
547         String cache = DocumentUtil.getValue(doc, fessConfig.getIndexFieldCache(), String.class);
548         if (cache != null) {
549             final String mimetype = DocumentUtil.getValue(doc, fessConfig.getIndexFieldMimetype(), String.class);
550             if (!ComponentUtil.getFessConfig().isHtmlMimetypeForCache(mimetype)) {
551                 cache = StringEscapeUtils.escapeHtml4(cache);
552             }
553             cache = ComponentUtil.getPathMappingHelper().replaceUrls(cache);
554             if (queries != null && queries.length > 0) {
555                 doc.put(HL_CACHE, replaceHighlightQueries(cache, queries));
556             } else {
557                 doc.put(HL_CACHE, cache);
558             }
559         } else {
560             doc.put(fessConfig.getIndexFieldCache(), StringUtil.EMPTY);
561             doc.put(HL_CACHE, StringUtil.EMPTY);
562         }
563 
564         try {
565             final Template template = handlebars.compile(cacheTemplateName);
566             final Context hbsContext = Context.newContext(doc);
567             return template.apply(hbsContext);
568         } catch (final Exception e) {
569             logger.warn("Failed to create a cache response.", e);
570         }
571 
572         return null;
573     }
574 
575     protected String replaceHighlightQueries(final String cache, final String[] queries) {
576         final StringBuffer buf = new StringBuffer(cache.length() + 100);
577         final StringBuffer segBuf = new StringBuffer(1000);
578         final Pattern p = Pattern.compile("<[^>]+>");
579         final Matcher m = p.matcher(cache);
580         final String[] regexQueries = new String[queries.length];
581         final String[] hlQueries = new String[queries.length];
582         for (int i = 0; i < queries.length; i++) {
583             regexQueries[i] = Pattern.quote(queries[i]);
584             hlQueries[i] = highlightTagPre + queries[i] + highlightTagPost;
585         }
586         while (m.find()) {
587             segBuf.setLength(0);
588             m.appendReplacement(segBuf, StringUtil.EMPTY);
589             String segment = segBuf.toString();
590             for (int i = 0; i < queries.length; i++) {
591                 segment = Pattern.compile(regexQueries[i], Pattern.CASE_INSENSITIVE).matcher(segment).replaceAll(hlQueries[i]);
592             }
593             buf.append(segment);
594             buf.append(m.group(0));
595         }
596         segBuf.setLength(0);
597         m.appendTail(segBuf);
598         String segment = segBuf.toString();
599         for (int i = 0; i < queries.length; i++) {
600             segment = Pattern.compile(regexQueries[i], Pattern.CASE_INSENSITIVE).matcher(segment).replaceAll(hlQueries[i]);
601         }
602         buf.append(segment);
603         return buf.toString();
604     }
605 
606     public Object getSitePath(final Map<String, Object> docMap) {
607         final FessConfig fessConfig = ComponentUtil.getFessConfig();
608         final Object urlLink = docMap.get(fessConfig.getResponseFieldUrlLink());
609         if (urlLink != null) {
610             final String returnUrl;
611             final String url = urlLink.toString();
612             if (LOCAL_PATH_PATTERN.matcher(url).find() || SHARED_FOLDER_PATTERN.matcher(url).find()) {
613                 returnUrl = url.replaceFirst("^file:/+", "");
614             } else if (url.startsWith("file:")) {
615                 returnUrl = url.replaceFirst("^file:/+", "/");
616             } else {
617                 returnUrl = url.replaceFirst("^[a-zA-Z0-9]*:/+", "");
618             }
619             final int size = fessConfig.getResponseMaxSitePathLengthAsInteger();
620             if (size > -1) {
621                 return StringUtils.abbreviate(returnUrl, size);
622             } else {
623                 return returnUrl;
624             }
625         }
626         return null;
627     }
628 
629     public StreamResponse asContentResponse(final Map<String, Object> doc) {
630         if (logger.isDebugEnabled()) {
631             logger.debug("writing the content of: {}", doc);
632         }
633         final FessConfig fessConfig = ComponentUtil.getFessConfig();
634         final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
635         final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
636         if (configId == null) {
637             throw new FessSystemException("configId is null.");
638         }
639         if (configId.length() < 2) {
640             throw new FessSystemException("Invalid configId: " + configId);
641         }
642         final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
643         if (config == null) {
644             throw new FessSystemException("No crawlingConfig: " + configId);
645         }
646         final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldUrl(), String.class);
647         final CrawlerClientFactory crawlerClientFactory =
648                 config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
649         final CrawlerClient client = crawlerClientFactory.getClient(url);
650         if (client == null) {
651             throw new FessSystemException("No CrawlerClient: " + configId + ", url: " + url);
652         }
653         return writeContent(configId, url, client);
654     }
655 
656     protected StreamResponse writeContent(final String configId, final String url, final CrawlerClient client) {
657         final StreamResponse response = new StreamResponse(StringUtil.EMPTY);
658         final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(url).build());
659         if (responseData.getHttpStatusCode() == 404) {
660             response.httpStatus(responseData.getHttpStatusCode());
661             CloseableUtil.closeQuietly(responseData);
662             return response;
663         }
664         writeFileName(response, responseData);
665         writeContentType(response, responseData);
666         writeNoCache(response, responseData);
667         response.stream(out -> {
668             try (final InputStream is = new BufferedInputStream(responseData.getResponseBody())) {
669                 out.write(is);
670             } catch (final IOException e) {
671                 if (!(e.getCause() instanceof ClientAbortException)) {
672                     throw new FessSystemException("Failed to write a content. configId: " + configId + ", url: " + url, e);
673                 }
674             } finally {
675                 CloseableUtil.closeQuietly(responseData);
676             }
677             if (logger.isDebugEnabled()) {
678                 logger.debug("Finished to write {}", url);
679             }
680         });
681         return response;
682     }
683 
684     protected void writeNoCache(final StreamResponse response, final ResponseData responseData) {
685         response.header("Pragma", "no-cache");
686         response.header("Cache-Control", "no-cache");
687         response.header("Expires", "Thu, 01 Dec 1994 16:00:00 GMT");
688     }
689 
690     protected void writeFileName(final StreamResponse response, final ResponseData responseData) {
691         String charset = responseData.getCharSet();
692         if (charset == null) {
693             charset = Constants.UTF_8;
694         }
695         final String name;
696         final String url = responseData.getUrl();
697         final int pos = url.lastIndexOf('/');
698         try {
699             if (pos >= 0 && pos + 1 < url.length()) {
700                 name = URLDecoder.decode(url.substring(pos + 1), charset);
701             } else {
702                 name = URLDecoder.decode(url, charset);
703             }
704 
705             final String contentDispositionType;
706             if (inlineMimeTypeSet.contains(responseData.getMimeType())) {
707                 contentDispositionType = "inline";
708             } else {
709                 contentDispositionType = "attachment";
710             }
711 
712             final String encodedName = URLEncoder.encode(name, Constants.UTF_8).replace("+", "%20");
713             response.header(CONTENT_DISPOSITION, contentDispositionType + "; filename=\"" + name + "\"; filename*=utf-8''" + encodedName);
714         } catch (final Exception e) {
715             logger.warn("Failed to write a filename: " + responseData, e);
716         }
717     }
718 
719     protected void writeContentType(final StreamResponse response, final ResponseData responseData) {
720         final String mimeType = responseData.getMimeType();
721         if (logger.isDebugEnabled()) {
722             logger.debug("mimeType: {}", mimeType);
723         }
724         if (mimeType == null) {
725             response.contentTypeOctetStream();
726             return;
727         }
728         if (mimeType.startsWith("text/")) {
729             final String charset = LaResponseUtil.getResponse().getCharacterEncoding();
730             if (charset != null) {
731                 response.contentType(mimeType + "; charset=" + charset);
732                 return;
733             }
734         }
735         response.contentType(mimeType);
736     }
737 
738     public String getClientIp(final HttpServletRequest request) {
739         final String value = request.getHeader("x-forwarded-for");
740         if (StringUtil.isNotBlank(value)) {
741             return value;
742         } else {
743             return request.getRemoteAddr();
744         }
745     }
746 
747     public FacetResponse getCachedFacetResponse(final String query) {
748         final OptionalThing<FessUserBean> userBean = ComponentUtil.getComponent(FessLoginAssist.class).getSavedUserBean();
749         final String permissionKey =
750                 userBean.map(
751                         user -> StreamUtil.stream(user.getPermissions()).get(
752                                 stream -> stream.sorted().distinct().collect(Collectors.joining("\n")))).orElse(StringUtil.EMPTY);
753 
754         try {
755             return facetCache.get(query + "\n" + permissionKey, () -> {
756                 final SearchHelper searchHelper = ComponentUtil.getSearchHelper();
757                 final SearchFormearchForm.html#SearchForm">SearchForm params = new SearchForm() {
758                     @Override
759                     public int getPageSize() {
760                         return 0;
761                     }
762 
763                     @Override
764                     public int getStartPosition() {
765                         return 0;
766                     }
767                 };
768                 params.q = query;
769                 final SearchRenderDatarData.html#SearchRenderData">SearchRenderData data = new SearchRenderData();
770                 searchHelper.search(params, data, userBean);
771                 if (logger.isDebugEnabled()) {
772                     logger.debug("loaded facet data: {}", data);
773                 }
774                 return data.getFacetResponse();
775             });
776         } catch (final ExecutionException e) {
777             throw new FessSystemException("Cannot load facet from cache.", e);
778         }
779     }
780 
781     public boolean isUseSession() {
782         return useSession;
783     }
784 
785     public void setUseSession(final boolean useSession) {
786         this.useSession = useSession;
787     }
788 
789     public void addInitFacetParam(final String key, final String value) {
790         initFacetParamMap.put(value, key);
791     }
792 
793     public Map<String, String> getInitFacetParamMap() {
794         return initFacetParamMap;
795     }
796 
797     public void addInitGeoParam(final String key, final String value) {
798         initGeoParamMap.put(value, key);
799     }
800 
801     public Map<String, String> getInitGeoParamMap() {
802         return initGeoParamMap;
803     }
804 
805     public void addFacetQueryView(final FacetQueryView facetQueryView) {
806         facetQueryViewList.add(facetQueryView);
807     }
808 
809     public List<FacetQueryView> getFacetQueryViewList() {
810         return facetQueryViewList;
811     }
812 
813     public void addInlineMimeType(final String mimeType) {
814         inlineMimeTypeSet.add(mimeType);
815     }
816 
817     public ActionHook getActionHook() {
818         return actionHook;
819     }
820 
821     public void setActionHook(final ActionHook actionHook) {
822         this.actionHook = actionHook;
823     }
824 
825     public static class ActionHook {
826 
827         public ActionResponse godHandPrologue(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
828             return func.apply(runtime);
829         }
830 
831         public ActionResponse godHandMonologue(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
832             return func.apply(runtime);
833         }
834 
835         public void godHandEpilogue(final ActionRuntime runtime, final Consumer<ActionRuntime> consumer) {
836             consumer.accept(runtime);
837         }
838 
839         public ActionResponse hookBefore(final ActionRuntime runtime, final Function<ActionRuntime, ActionResponse> func) {
840             return func.apply(runtime);
841         }
842 
843         public void hookFinally(final ActionRuntime runtime, final Consumer<ActionRuntime> consumer) {
844             consumer.accept(runtime);
845         }
846     }
847 
848     public void setEncodeUrlLink(final boolean encodeUrlLink) {
849         this.encodeUrlLink = encodeUrlLink;
850     }
851 
852     public void setUrlLinkEncoding(final String urlLinkEncoding) {
853         this.urlLinkEncoding = urlLinkEncoding;
854     }
855 
856     public void setOriginalHighlightTagPre(final String originalHighlightTagPre) {
857         this.originalHighlightTagPre = originalHighlightTagPre;
858     }
859 
860     public void setOriginalHighlightTagPost(final String originalHighlightTagPost) {
861         this.originalHighlightTagPost = originalHighlightTagPost;
862     }
863 
864     public void setCacheTemplateName(final String cacheTemplateName) {
865         this.cacheTemplateName = cacheTemplateName;
866     }
867 
868     public void setFacetCacheDuration(final long facetCacheDuration) {
869         this.facetCacheDuration = facetCacheDuration;
870     }
871 }