View Javadoc
1   /*
2    * Copyright 2012-2017 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.mylasta.direction;
17  
18  import static org.codelibs.core.stream.StreamUtil.split;
19  import static org.codelibs.core.stream.StreamUtil.stream;
20  
21  import java.net.Authenticator;
22  import java.net.InetSocketAddress;
23  import java.net.PasswordAuthentication;
24  import java.net.Proxy;
25  import java.net.Proxy.Type;
26  import java.net.SocketAddress;
27  import java.util.ArrayList;
28  import java.util.Collections;
29  import java.util.Date;
30  import java.util.Enumeration;
31  import java.util.HashSet;
32  import java.util.List;
33  import java.util.Locale;
34  import java.util.Map;
35  import java.util.Set;
36  import java.util.concurrent.ConcurrentHashMap;
37  import java.util.function.Function;
38  import java.util.regex.Matcher;
39  import java.util.regex.Pattern;
40  import java.util.stream.Collectors;
41  
42  import javax.naming.directory.Attribute;
43  import javax.naming.directory.BasicAttribute;
44  import javax.servlet.http.HttpSession;
45  
46  import org.codelibs.core.exception.ClassNotFoundRuntimeException;
47  import org.codelibs.core.lang.StringUtil;
48  import org.codelibs.core.misc.Pair;
49  import org.codelibs.core.misc.Tuple3;
50  import org.codelibs.fess.Constants;
51  import org.codelibs.fess.exception.FessSystemException;
52  import org.codelibs.fess.helper.PermissionHelper;
53  import org.codelibs.fess.mylasta.action.FessUserBean;
54  import org.codelibs.fess.taglib.FessFunctions;
55  import org.codelibs.fess.util.ComponentUtil;
56  import org.codelibs.fess.util.PrunedTag;
57  import org.dbflute.optional.OptionalThing;
58  import org.elasticsearch.action.search.SearchRequestBuilder;
59  import org.elasticsearch.search.sort.SortBuilder;
60  import org.elasticsearch.search.sort.SortBuilders;
61  import org.elasticsearch.search.sort.SortOrder;
62  import org.lastaflute.job.LaJob;
63  import org.lastaflute.job.subsidiary.JobConcurrentExec;
64  import org.lastaflute.web.response.next.HtmlNext;
65  import org.lastaflute.web.util.LaRequestUtil;
66  import org.lastaflute.web.validation.RequiredValidator;
67  import org.lastaflute.web.validation.theme.typed.DoubleTypeValidator;
68  import org.lastaflute.web.validation.theme.typed.FloatTypeValidator;
69  import org.lastaflute.web.validation.theme.typed.IntegerTypeValidator;
70  import org.lastaflute.web.validation.theme.typed.LongTypeValidator;
71  
72  public interface FessProp {
73  
74      public static final String THUMBNAIL_HTML_IMAGE_EXCLUDE_EXTENSIONS = "ThumbnailHtmlImageExcludeExtensions";
75  
76      public static final String VIRTUAL_HOST_VALUE = "VirtualHostValue";
77  
78      public static final String QUERY_DEFAULT_LANGUAGES = "queryDefaultLanguages";
79  
80      public static final String HTML_PROXY = "httpProxy";
81  
82      public static final String CRAWLER_FAILURE_URL_STATUS_CODES = "crawlerFailureUrlStatusCodes";
83  
84      public static final String VIRTUAL_HOST_HEADERS = "virtualHostHeaders";
85  
86      public static final String QUERY_COLLAPSE_INNER_HITS_SORTS = "queryCollapseInnerHitsSorts";
87  
88      public static final String USER_CODE_PATTERN = "userCodePattern";
89  
90      public static final String API_ADMIN_ACCESS_PERMISSION_SET = "apiAdminAccessPermissionSet";
91  
92      public static final String CRAWLER_DOCUMENT_SPACE_CHARS = "crawlerDocumentSpaceChars";
93  
94      public static final String INDEX_ADMIN_ARRAY_FIELD_SET = "indexAdminArrayFieldSet";
95  
96      public static final String INDEX_ADMIN_DATE_FIELD_SET = "indexAdminDateFieldSet";
97  
98      public static final String INDEX_ADMIN_INTEGER_FIELD_SET = "indexAdminIntegerFieldSet";
99  
100     public static final String INDEX_ADMIN_LONG_FIELD_SET = "indexAdminLongFieldSet";
101 
102     public static final String INDEX_ADMIN_FLOAT_FIELD_SET = "indexAdminFloatFieldSet";
103 
104     public static final String INDEX_ADMIN_DOUBLE_FIELD_SET = "indexAdminDoubleFieldSet";
105 
106     public static final String OIC_DEFAULT_ROLES = "oicDefaultRoles";
107 
108     public static final String OIC_DEFAULT_GROUPS = "oicDefaultGroups";
109 
110     public static final String AUTHENTICATION_ADMIN_ROLES = "authenticationAdminRoles";
111 
112     public static final String SEARCH_GUEST_PERMISSION_LIST = "searchGuestPermissionList";
113 
114     public static final String SUGGEST_SEARCH_LOG_PERMISSIONS = "suggestSearchLogPermissions";
115 
116     public static final String GROUP_VALUE_PREFIX = "group:";
117 
118     public static final String ROLE_VALUE_PREFIX = "role:";
119 
120     public static final String DEFAULT_SORT_VALUES = "defaultSortValues";
121 
122     public static final String DEFAULT_LABEL_VALUES = "defaultLabelValues";
123 
124     public static final String VIRTUAL_HOST_VALUES = "virtualHostValues";
125 
126     public static final String QUERY_LANGUAGE_MAPPING = "queryLanguageMapping";
127 
128     public static final String CRAWLER_METADATA_NAME_MAPPING = "crawlerMetadataNameMapping";
129 
130     public static final String CRAWLER_METADATA_CONTENT_EXCLUDES = "crawlerMetadataContentExcludes";
131 
132     public static final Map<String, Object> propMap = new ConcurrentHashMap<>();
133 
134     //
135     // system.properties
136     //
137 
138     public default void storeSystemProperties() {
139         ComponentUtil.getSystemProperties().store();
140     }
141 
142     public default String getSystemProperty(final String key) {
143         return ComponentUtil.getSystemProperties().getProperty(key);
144     }
145 
146     public default String getSystemProperty(final String key, final String defaultValue) {
147         return ComponentUtil.getSystemProperties().getProperty(key, defaultValue);
148     }
149 
150     public default void setSystemProperty(final String key, final String value) {
151         if (value != null) {
152             ComponentUtil.getSystemProperties().setProperty(key, value);
153         } else {
154             ComponentUtil.getSystemProperties().remove(key);
155         }
156     }
157 
158     public default boolean getSystemPropertyAsBoolean(final String key, final boolean defaultValue) {
159         return Constants.TRUE.equalsIgnoreCase(getSystemProperty(key, defaultValue ? Constants.TRUE : Constants.FALSE));
160     }
161 
162     public default void setSystemPropertyAsBoolean(final String key, final boolean value) {
163         setSystemProperty(key, value ? Constants.TRUE : Constants.FALSE);
164     }
165 
166     public default int getSystemPropertyAsInt(final String key, final int defaultValue) {
167         final String value = getSystemProperty(key);
168         if (value != null) {
169             try {
170                 return Integer.valueOf(value);
171             } catch (final NumberFormatException e) {
172                 // ignore
173             }
174         }
175         return defaultValue;
176     }
177 
178     public default void setSystemPropertyAsInt(final String key, final int value) {
179         setSystemProperty(key, Integer.toString(value));
180     }
181 
182     public default boolean isWebDesignEditorEnabled() {
183         return getSystemPropertyAsBoolean(Constants.WEB_DESIGN_EDITOR_PROPERTY, true);
184     }
185 
186     public default boolean isSearchFileProxyEnabled() {
187         return getSystemPropertyAsBoolean(Constants.SEARCH_FILE_PROXY_PROPERTY, true);
188     }
189 
190     public default boolean isBrowserLocaleForSearchUsed() {
191         return getSystemPropertyAsBoolean(Constants.USE_BROWSER_LOCALE_FOR_SEARCH_PROPERTY, false);
192     }
193 
194     public default String[] getDefaultSortValues(final OptionalThing<FessUserBean> userBean) {
195         @SuppressWarnings("unchecked")
196         List<Pair<String, String>> list = (List<Pair<String, String>>) propMap.get(DEFAULT_SORT_VALUES);
197         if (list == null) {
198             final String value = getSystemProperty(Constants.DEFAULT_SORT_VALUE_PROPERTY);
199             if (StringUtil.isBlank(value)) {
200                 list = Collections.emptyList();
201             } else {
202                 final Set<String> keySet = new HashSet<>();
203                 list = split(value, "\n").get(stream -> stream.filter(StringUtil::isNotBlank).map(s -> {
204                     final String[] pair = s.split("=");
205                     if (pair.length == 1) {
206                         return new Pair<>(StringUtil.EMPTY, pair[0].trim());
207                     } else if (pair.length == 2) {
208                         String sortValue = pair[1].trim();
209                         if (StringUtil.isBlank(sortValue) || "score".equals(sortValue)) {
210                             sortValue = "score.desc";
211                         }
212                         return new Pair<>(pair[0].trim(), sortValue);
213                     }
214                     return null;
215                 }).filter(o -> o != null && keySet.add(o.getFirst())).collect(Collectors.toList()));
216             }
217             propMap.put(DEFAULT_SORT_VALUES, list);
218         }
219         return list
220                 .stream()
221                 .map(p -> {
222                     final String key = p.getFirst();
223                     if (StringUtil.isEmpty(key)) {
224                         return p.getSecond();
225                     }
226                     if (userBean.map(
227                             user -> stream(user.getRoles()).get(stream -> stream.anyMatch(s -> key.equals(ROLE_VALUE_PREFIX + s)))
228                                     || stream(user.getGroups()).get(stream -> stream.anyMatch(s -> key.equals(GROUP_VALUE_PREFIX + s))))
229                             .orElse(false)) {
230                         return p.getSecond();
231                     }
232                     return null;
233                 }).filter(StringUtil::isNotBlank).toArray(n -> new String[n]);
234     }
235 
236     public default void setDefaultSortValue(final String value) {
237         setSystemProperty(Constants.DEFAULT_SORT_VALUE_PROPERTY, value);
238         propMap.remove(DEFAULT_SORT_VALUES);
239     }
240 
241     public default String getDefaultSortValue() {
242         return getSystemProperty(Constants.DEFAULT_SORT_VALUE_PROPERTY, StringUtil.EMPTY);
243     }
244 
245     public default String[] getDefaultLabelValues(final OptionalThing<FessUserBean> userBean) {
246         @SuppressWarnings("unchecked")
247         Map<String, String> map = (Map<String, String>) propMap.get(DEFAULT_LABEL_VALUES);
248         if (map == null) {
249             final String value = getSystemProperty(Constants.DEFAULT_LABEL_VALUE_PROPERTY);
250             if (StringUtil.isBlank(value)) {
251                 map = Collections.emptyMap();
252             } else {
253                 final Set<String> keySet = new HashSet<>();
254                 map = split(value, "\n").get(stream -> stream.filter(StringUtil::isNotBlank).map(s -> {
255                     final String[] pair = s.split("=");
256                     if (pair.length == 1) {
257                         return new Pair<>(StringUtil.EMPTY, pair[0].trim());
258                     } else if (pair.length == 2) {
259                         return new Pair<>(pair[0].trim(), pair[1].trim());
260                     }
261                     return null;
262                 }).filter(o -> o != null && keySet.add(o.getFirst())).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
263             }
264             propMap.put(DEFAULT_LABEL_VALUES, map);
265         }
266         return map
267                 .entrySet()
268                 .stream()
269                 .map(e -> {
270                     final String key = e.getKey();
271                     if (StringUtil.isEmpty(key)) {
272                         return e.getValue();
273                     }
274                     if (userBean.map(
275                             user -> stream(user.getRoles()).get(stream -> stream.anyMatch(s -> key.equals(ROLE_VALUE_PREFIX + s)))
276                                     || stream(user.getGroups()).get(stream -> stream.anyMatch(s -> key.equals(GROUP_VALUE_PREFIX + s))))
277                             .orElse(false)) {
278                         return e.getValue();
279                     }
280                     return null;
281                 }).filter(StringUtil::isNotBlank).toArray(n -> new String[n]);
282     }
283 
284     public default void setDefaultLabelValue(final String value) {
285         setSystemProperty(Constants.DEFAULT_LABEL_VALUE_PROPERTY, value);
286         propMap.remove(DEFAULT_LABEL_VALUES);
287     }
288 
289     public default String getDefaultLabelValue() {
290         return getSystemProperty(Constants.DEFAULT_LABEL_VALUE_PROPERTY, StringUtil.EMPTY);
291     }
292 
293     public default void setVirtualHostValue(final String value) {
294         setSystemProperty(Constants.VIRTUAL_HOST_VALUE_PROPERTY, value);
295         propMap.remove(VIRTUAL_HOST_HEADERS);
296     }
297 
298     public default String getVirtualHostValue() {
299         return getSystemProperty(Constants.VIRTUAL_HOST_VALUE_PROPERTY, getVirtualHostHeaders());
300     }
301 
302     public default void setLoginRequired(final boolean value) {
303         setSystemPropertyAsBoolean(Constants.LOGIN_REQUIRED_PROPERTY, value);
304     }
305 
306     public default boolean isLoginRequired() {
307         return getSystemPropertyAsBoolean(Constants.LOGIN_REQUIRED_PROPERTY, false);
308     }
309 
310     public default void setResultCollapsed(final boolean value) {
311         setSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, value);
312     }
313 
314     public default boolean isResultCollapsed() {
315         return getSystemPropertyAsBoolean(Constants.RESULT_COLLAPSED_PROPERTY, false);
316     }
317 
318     public default void setLoginLinkEnabled(final boolean value) {
319         setSystemPropertyAsBoolean(Constants.LOGIN_LINK_ENALBED_PROPERTY, value);
320     }
321 
322     public default boolean isLoginLinkEnabled() {
323         return getSystemPropertyAsBoolean(Constants.LOGIN_LINK_ENALBED_PROPERTY, true);
324     }
325 
326     public default void setThumbnailEnabled(final boolean value) {
327         setSystemPropertyAsBoolean(Constants.THUMBNAIL_ENALBED_PROPERTY, value);
328     }
329 
330     public default boolean isThumbnailEnabled() {
331         return getSystemPropertyAsBoolean(Constants.THUMBNAIL_ENALBED_PROPERTY, false);
332     }
333 
334     public default void setIncrementalCrawling(final boolean value) {
335         setSystemPropertyAsBoolean(Constants.INCREMENTAL_CRAWLING_PROPERTY, value);
336     }
337 
338     public default boolean isIncrementalCrawling() {
339         return getSystemPropertyAsBoolean(Constants.INCREMENTAL_CRAWLING_PROPERTY, true);
340     }
341 
342     public default void setDayForCleanup(final int value) {
343         setSystemPropertyAsInt(Constants.DAY_FOR_CLEANUP_PROPERTY, value);
344     }
345 
346     public default int getDayForCleanup() {
347         return getSystemPropertyAsInt(Constants.DAY_FOR_CLEANUP_PROPERTY, Constants.DEFAULT_DAY_FOR_CLEANUP);
348     }
349 
350     public default void setCrawlingThreadCount(final int value) {
351         setSystemPropertyAsInt(Constants.CRAWLING_THREAD_COUNT_PROPERTY, value);
352     }
353 
354     public default int getCrawlingThreadCount() {
355         return getSystemPropertyAsInt(Constants.CRAWLING_THREAD_COUNT_PROPERTY, 5);
356     }
357 
358     public default void setSearchLog(final boolean value) {
359         setSystemPropertyAsBoolean(Constants.SEARCH_LOG_PROPERTY, value);
360     }
361 
362     public default boolean isSearchLog() {
363         return getSystemPropertyAsBoolean(Constants.SEARCH_LOG_PROPERTY, true);
364     }
365 
366     public default void setUserInfo(final boolean value) {
367         setSystemPropertyAsBoolean(Constants.USER_INFO_PROPERTY, value);
368     }
369 
370     public default boolean isUserInfo() {
371         return getSystemPropertyAsBoolean(Constants.USER_INFO_PROPERTY, true);
372     }
373 
374     public default void setUserFavorite(final boolean value) {
375         setSystemPropertyAsBoolean(Constants.USER_FAVORITE_PROPERTY, value);
376     }
377 
378     public default boolean isUserFavorite() {
379         return getSystemPropertyAsBoolean(Constants.USER_FAVORITE_PROPERTY, false);
380     }
381 
382     public default void setWebApiJson(final boolean value) {
383         setSystemPropertyAsBoolean(Constants.WEB_API_JSON_PROPERTY, value);
384     }
385 
386     public default boolean isWebApiJson() {
387         return getSystemPropertyAsBoolean(Constants.WEB_API_JSON_PROPERTY, true);
388     }
389 
390     public default boolean isWebApiGsa() {
391         return getSystemPropertyAsBoolean(Constants.WEB_API_GSA_PROPERTY, false);
392     }
393 
394     public default void setAppendQueryParameter(final boolean value) {
395         setSystemPropertyAsBoolean(Constants.APPEND_QUERY_PARAMETER_PROPERTY, value);
396     }
397 
398     public default boolean isAppendQueryParameter() {
399         return getSystemPropertyAsBoolean(Constants.APPEND_QUERY_PARAMETER_PROPERTY, false);
400     }
401 
402     public default void setIgnoreFailureType(final String value) {
403         setSystemProperty(Constants.IGNORE_FAILURE_TYPE_PROPERTY, value);
404     }
405 
406     public default String getIgnoreFailureType() {
407         return getSystemProperty(Constants.IGNORE_FAILURE_TYPE_PROPERTY, Constants.DEFAULT_IGNORE_FAILURE_TYPE);
408     }
409 
410     public default void setFailureCountThreshold(final int value) {
411         setSystemPropertyAsInt(Constants.FAILURE_COUNT_THRESHOLD_PROPERTY, value);
412     }
413 
414     public default int getFailureCountThreshold() {
415         return getSystemPropertyAsInt(Constants.FAILURE_COUNT_THRESHOLD_PROPERTY, Constants.DEFAULT_FAILURE_COUNT);
416     }
417 
418     public default void setWebApiPopularWord(final boolean value) {
419         setSystemPropertyAsBoolean(Constants.WEB_API_POPULAR_WORD_PROPERTY, value);
420     }
421 
422     public default boolean isWebApiPopularWord() {
423         return getSystemPropertyAsBoolean(Constants.WEB_API_POPULAR_WORD_PROPERTY, true);
424     }
425 
426     public default void setCsvFileEncoding(final String value) {
427         setSystemProperty(Constants.CSV_FILE_ENCODING_PROPERTY, value);
428     }
429 
430     public default String getCsvFileEncoding() {
431         return getSystemProperty(Constants.CSV_FILE_ENCODING_PROPERTY, Constants.UTF_8);
432     }
433 
434     public default void setPurgeSearchLogDay(final int value) {
435         setSystemPropertyAsInt(Constants.PURGE_SEARCH_LOG_DAY_PROPERTY, value);
436     }
437 
438     public default int getPurgeSearchLogDay() {
439         return getSystemPropertyAsInt(Constants.PURGE_SEARCH_LOG_DAY_PROPERTY, Integer.parseInt(Constants.DEFAULT_PURGE_DAY));
440     }
441 
442     public default void setPurgeJobLogDay(final int value) {
443         setSystemPropertyAsInt(Constants.PURGE_JOB_LOG_DAY_PROPERTY, value);
444     }
445 
446     public default int getPurgeJobLogDay() {
447         return getSystemPropertyAsInt(Constants.PURGE_JOB_LOG_DAY_PROPERTY, Integer.parseInt(Constants.DEFAULT_PURGE_DAY));
448     }
449 
450     public default void setPurgeUserInfoDay(final int value) {
451         setSystemPropertyAsInt(Constants.PURGE_USER_INFO_DAY_PROPERTY, value);
452     }
453 
454     public default int getPurgeUserInfoDay() {
455         return getSystemPropertyAsInt(Constants.PURGE_USER_INFO_DAY_PROPERTY, Integer.parseInt(Constants.DEFAULT_PURGE_DAY));
456     }
457 
458     public default void setPurgeByBots(final String value) {
459         setSystemProperty(Constants.PURGE_BY_BOTS_PROPERTY, value);
460     }
461 
462     public default String getPurgeByBots() {
463         return getSystemProperty(Constants.PURGE_BY_BOTS_PROPERTY, Constants.DEFAULT_PURGE_BY_BOTS);
464     }
465 
466     public default void setNotificationTo(final String value) {
467         setSystemProperty(Constants.NOTIFICATION_TO_PROPERTY, value);
468     }
469 
470     public default String getNotificationTo() {
471         return getSystemProperty(Constants.NOTIFICATION_TO_PROPERTY, StringUtil.EMPTY);
472     }
473 
474     public default void setSuggestSearchLog(final boolean value) {
475         setSystemPropertyAsBoolean(Constants.SUGGEST_SEARCH_LOG_PROPERTY, value);
476     }
477 
478     public default boolean isSuggestSearchLog() {
479         return getSystemPropertyAsBoolean(Constants.SUGGEST_SEARCH_LOG_PROPERTY, true);
480     }
481 
482     public default void setSuggestDocuments(final boolean value) {
483         setSystemPropertyAsBoolean(Constants.SUGGEST_DOCUMENTS_PROPERTY, value);
484     }
485 
486     public default boolean isSuggestDocuments() {
487         return getSystemPropertyAsBoolean(Constants.SUGGEST_DOCUMENTS_PROPERTY, true);
488     }
489 
490     public default void setPurgeSuggestSearchLogDay(final int value) {
491         setSystemPropertyAsInt(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY, value);
492     }
493 
494     public default int getPurgeSuggestSearchLogDay() {
495         return getSystemPropertyAsInt(Constants.PURGE_SUGGEST_SEARCH_LOG_DAY_PROPERTY,
496                 Integer.parseInt(Constants.DEFAULT_SUGGEST_PURGE_DAY));
497     }
498 
499     public default void setLdapInitialContextFactory(final String value) {
500         setSystemProperty(Constants.LDAP_INITIAL_CONTEXT_FACTORY, value);
501     }
502 
503     public default String getLdapInitialContextFactory() {
504         return getSystemProperty(Constants.LDAP_INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory");
505     }
506 
507     public default void setLdapSecurityAuthentication(final String value) {
508         setSystemProperty(Constants.LDAP_SECURITY_AUTHENTICATION, value);
509     }
510 
511     public default String getLdapSecurityAuthentication() {
512         return getSystemProperty(Constants.LDAP_SECURITY_AUTHENTICATION, "simple");
513     }
514 
515     public default void setLdapProviderUrl(final String value) {
516         setSystemProperty(Constants.LDAP_PROVIDER_URL, value);
517     }
518 
519     public default String getLdapProviderUrl() {
520         return getSystemProperty(Constants.LDAP_PROVIDER_URL);
521     }
522 
523     public default void setLdapSecurityPrincipal(final String value) {
524         setSystemProperty(Constants.LDAP_SECURITY_PRINCIPAL, value);
525     }
526 
527     public default String getLdapMemberofAttribute() {
528         return getSystemProperty(Constants.LDAP_MEMBEROF_ATTRIBUTE, "memberOf");
529     }
530 
531     public default void setLdapMemberofAttribute(final String value) {
532         setSystemProperty(Constants.LDAP_MEMBEROF_ATTRIBUTE, value);
533     }
534 
535     Integer getLdapMaxUsernameLengthAsInteger();
536 
537     public default String getLdapSecurityPrincipal(final String username) {
538         final String value;
539         final int maxLength = getLdapMaxUsernameLengthAsInteger().intValue();
540         if (username == null) {
541             value = StringUtil.EMPTY;
542         } else if (maxLength >= 0 && username.length() > maxLength) {
543             value = username.substring(0, maxLength);
544         } else {
545             value = username;
546         }
547         return String.format(getSystemProperty(Constants.LDAP_SECURITY_PRINCIPAL, StringUtil.EMPTY), value);
548     }
549 
550     public default String getLdapSecurityPrincipal() {
551         return getSystemProperty(Constants.LDAP_SECURITY_PRINCIPAL);
552     }
553 
554     public default void setLdapAdminSecurityPrincipal(final String value) {
555         setSystemProperty(Constants.LDAP_ADMIN_SECURITY_PRINCIPAL, value);
556     }
557 
558     public default String getLdapAdminSecurityPrincipal() {
559         return getSystemProperty(Constants.LDAP_ADMIN_SECURITY_PRINCIPAL);
560     }
561 
562     public default void setLdapAdminSecurityCredentials(final String value) {
563         setSystemProperty(Constants.LDAP_ADMIN_SECURITY_CREDENTIALS,
564                 Constants.CIPHER_PREFIX + ComponentUtil.getPrimaryCipher().encrypt(value));
565     }
566 
567     public default String getLdapAdminSecurityCredentials() {
568         final String value = getSystemProperty(Constants.LDAP_ADMIN_SECURITY_CREDENTIALS);
569         if (StringUtil.isNotBlank(value) && value.startsWith(Constants.CIPHER_PREFIX)) {
570             return ComponentUtil.getPrimaryCipher().decrypt(value.substring(Constants.CIPHER_PREFIX.length()));
571         }
572         return value;
573     }
574 
575     public default void setLdapBaseDn(final String value) {
576         setSystemProperty(Constants.LDAP_BASE_DN, value);
577     }
578 
579     public default String getLdapBaseDn() {
580         return getSystemProperty(Constants.LDAP_BASE_DN);
581     }
582 
583     public default void setLdapAccountFilter(final String value) {
584         setSystemProperty(Constants.LDAP_ACCOUNT_FILTER, value);
585     }
586 
587     public default String getLdapAccountFilter() {
588         return getSystemProperty(Constants.LDAP_ACCOUNT_FILTER);
589     }
590 
591     public default void setNotificationLogin(final String value) {
592         setSystemProperty(Constants.NOTIFICATION_LOGIN, value);
593     }
594 
595     public default String getNotificationLogin() {
596         return getSystemProperty(Constants.NOTIFICATION_LOGIN, StringUtil.EMPTY);
597     }
598 
599     public default void setNotificationSearchTop(final String value) {
600         setSystemProperty(Constants.NOTIFICATION_SEARCH_TOP, value);
601     }
602 
603     public default String getNotificationSearchTop() {
604         return getSystemProperty(Constants.NOTIFICATION_SEARCH_TOP, StringUtil.EMPTY);
605     }
606 
607     //
608     // fess_*.properties
609     //
610 
611     String getAuthenticationAdminRoles();
612 
613     public default String[] getAuthenticationAdminRolesAsArray() {
614         String[] roles = (String[]) propMap.get(AUTHENTICATION_ADMIN_ROLES);
615         if (roles == null) {
616             roles = getAuthenticationAdminRoles().split(",");
617             propMap.put(AUTHENTICATION_ADMIN_ROLES, roles);
618         }
619         return roles;
620     }
621 
622     String getJvmCrawlerOptions();
623 
624     public default String[] getJvmCrawlerOptionsAsArray() {
625         return getJvmCrawlerOptions().split("\n");
626     }
627 
628     String getJvmSuggestOptions();
629 
630     public default String[] getJvmSuggestOptionsAsArray() {
631         return getJvmSuggestOptions().split("\n");
632     }
633 
634     String getCrawlerDocumentHtmlPrunedTags();
635 
636     public default PrunedTag[] getCrawlerDocumentHtmlPrunedTagsAsArray() {
637         PrunedTag[] tags = (PrunedTag[]) propMap.get("crawlerDocumentHtmlPrunedTags");
638         if (tags == null) {
639             tags = split(getCrawlerDocumentHtmlPrunedTags(), ",").get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
640                 final Pattern pattern = Pattern.compile("(\\w+)(\\[[^\\]]+\\])?(\\.\\w+)?(#\\w+)?");
641                 final Matcher matcher = pattern.matcher(v.trim());
642                 if (matcher.matches()) {
643                     final PrunedTag tag = new PrunedTag(matcher.group(1));
644                     if (matcher.group(2) != null) {
645                         final String attrPair = matcher.group(2).substring(1, matcher.group(2).length() - 1);
646                         final Matcher equalMatcher = Pattern.compile("(\\w+)=(\\w+)").matcher(attrPair);
647                         if (equalMatcher.matches()) {
648                             tag.setAttr(equalMatcher.group(1), equalMatcher.group(2));
649                         }
650                     }
651                     if (matcher.group(3) != null) {
652                         tag.setCss(matcher.group(3).substring(1));
653                     }
654                     if (matcher.group(4) != null) {
655                         tag.setId(matcher.group(4).substring(1));
656                     }
657                     return tag;
658                 }
659                 throw new FessSystemException("Invalid pruned tag: " + v);
660             }).toArray(n -> new PrunedTag[n]));
661             propMap.put("crawlerDocumentHtmlPrunedTags", tags);
662         }
663         return tags;
664     }
665 
666     String getCrawlerDocumentCacheHtmlMimetypes();
667 
668     public default boolean isHtmlMimetypeForCache(final String mimetype) {
669         final String[] mimetypes = getCrawlerDocumentCacheHtmlMimetypes().split(",");
670         if (mimetypes.length == 1 && StringUtil.isBlank(mimetypes[0])) {
671             return true;
672         }
673         return stream(mimetypes).get(stream -> stream.anyMatch(s -> s.equalsIgnoreCase(mimetype)));
674     }
675 
676     String getCrawlerDocumentCacheSupportedMimetypes();
677 
678     public default boolean isSupportedDocumentCacheMimetypes(final String mimetype) {
679         final String[] mimetypes = getCrawlerDocumentCacheSupportedMimetypes().split(",");
680         if (mimetypes.length == 1 && StringUtil.isBlank(mimetypes[0])) {
681             return true;
682         }
683         return stream(mimetypes).get(stream -> stream.anyMatch(s -> s.equalsIgnoreCase(mimetype)));
684     }
685 
686     String getIndexerClickCountEnabled();
687 
688     public default boolean getIndexerClickCountEnabledAsBoolean() {
689         return Constants.TRUE.equalsIgnoreCase(getIndexerClickCountEnabled());
690     }
691 
692     String getIndexerFavoriteCountEnabled();
693 
694     public default boolean getIndexerFavoriteCountEnabledAsBoolean() {
695         return Constants.TRUE.equalsIgnoreCase(getIndexerFavoriteCountEnabled());
696     }
697 
698     String getApiAccessTokenRequired();
699 
700     public default boolean getApiAccessTokenRequiredAsBoolean() {
701         return Constants.TRUE.equalsIgnoreCase(getApiAccessTokenRequired());
702     }
703 
704     String getIndexerThreadDumpEnabled();
705 
706     public default boolean getIndexerThreadDumpEnabledAsBoolean() {
707         return Constants.TRUE.equalsIgnoreCase(getIndexerThreadDumpEnabled());
708     }
709 
710     String getIndexBackupTargets();
711 
712     String getIndexBackupLogTargets();
713 
714     public default String[] getIndexBackupAllTargets() {
715         return split(getIndexBackupTargets() + "," + getIndexBackupLogTargets(), ",").get(
716                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).toArray(n -> new String[n]));
717     }
718 
719     String getJobSystemJobIds();
720 
721     public default boolean isSystemJobId(final String id) {
722         if (StringUtil.isBlank(getJobSystemJobIds())) {
723             return false;
724         }
725         return split(getJobSystemJobIds(), ",").get(stream -> stream.anyMatch(s -> s.equals(id)));
726     }
727 
728     String getSmbAvailableSidTypes();
729 
730     public default boolean isAvailableSmbSidType(final int sidType) {
731         if (StringUtil.isBlank(getSmbAvailableSidTypes())) {
732             return false;
733         }
734         final String value = Integer.toString(sidType);
735         return split(getSmbAvailableSidTypes(), ",").get(stream -> stream.anyMatch(s -> s.equals(value)));
736     }
737 
738     String getSupportedLanguages();
739 
740     public default String[] getSupportedLanguagesAsArray() {
741         return split(getSupportedLanguages(), ",").get(stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
742     }
743 
744     String getOnlineHelpSupportedLangs();
745 
746     public default boolean isOnlineHelpSupportedLang(final String lang) {
747         if (StringUtil.isBlank(getOnlineHelpSupportedLangs())) {
748             return false;
749         }
750         return split(getOnlineHelpSupportedLangs(), ",").get(stream -> stream.filter(StringUtil::isNotBlank).anyMatch(s -> s.equals(lang)));
751     }
752 
753     String getSupportedUploadedJsExtentions();
754 
755     public default String[] getSupportedUploadedJsExtentionsAsArray() {
756         return split(getSupportedUploadedJsExtentions(), ",").get(
757                 stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
758     }
759 
760     String getSupportedUploadedCssExtentions();
761 
762     public default String[] getSupportedUploadedCssExtentionsAsArray() {
763         return split(getSupportedUploadedCssExtentions(), ",").get(
764                 stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
765     }
766 
767     String getSupportedUploadedMediaExtentions();
768 
769     public default String[] getSupportedUploadedMediaExtentionsAsArray() {
770         return split(getSupportedUploadedMediaExtentions(), ",").get(
771                 stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
772     }
773 
774     String getJobTemplateTitleWeb();
775 
776     String getJobTemplateTitleFile();
777 
778     String getJobTemplateTitleData();
779 
780     public default String getJobTemplateTitle(final String type) {
781         if (Constants.WEB_CRAWLER_TYPE.equals(type)) {
782             return getJobTemplateTitleWeb();
783         } else if (Constants.FILE_CRAWLER_TYPE.equals(type)) {
784             return getJobTemplateTitleFile();
785         } else if (Constants.DATA_CRAWLER_TYPE.equals(type)) {
786             return getJobTemplateTitleData();
787         }
788         return "None";
789     }
790 
791     String getSchedulerJobClass();
792 
793     public default Class<? extends LaJob> getSchedulerJobClassAsClass() {
794         try {
795             @SuppressWarnings("unchecked")
796             final Class<? extends LaJob> clazz = (Class<? extends LaJob>) Class.forName(getSchedulerJobClass());
797             return clazz;
798         } catch (final ClassNotFoundException e) {
799             throw new ClassNotFoundRuntimeException(e);
800         }
801     }
802 
803     String getSchedulerConcurrentExecMode();
804 
805     public default JobConcurrentExec getSchedulerConcurrentExecModeAsEnum() {
806         return JobConcurrentExec.valueOf(getSchedulerConcurrentExecMode());
807     }
808 
809     String getCrawlerMetadataContentExcludes();
810 
811     public default boolean isCrawlerMetadataContentIncluded(final String name) {
812         Pattern[] patterns = (Pattern[]) propMap.get(CRAWLER_METADATA_CONTENT_EXCLUDES);
813         if (patterns == null) {
814             patterns =
815                     split(getCrawlerMetadataContentExcludes(), ",").get(
816                             stream -> stream.filter(StringUtil::isNotBlank).map(v -> Pattern.compile(v)).toArray(n -> new Pattern[n]));
817             propMap.put(CRAWLER_METADATA_CONTENT_EXCLUDES, patterns);
818         }
819         return !stream(patterns).get(stream -> stream.anyMatch(p -> p.matcher(name).matches()));
820     }
821 
822     String getCrawlerMetadataNameMapping();
823 
824     public default Pair<String, String> getCrawlerMetadataNameMapping(final String name) {
825         @SuppressWarnings("unchecked")
826         Map<String, Pair<String, String>> params = (Map<String, Pair<String, String>>) propMap.get(CRAWLER_METADATA_NAME_MAPPING);
827         if (params == null) {
828             params = split(getCrawlerMetadataNameMapping(), "\n").get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
829                 final String[] values = v.split("=");
830                 if (values.length == 2) {
831                     final String[] subValues = values[1].split(":");
832                     if (subValues.length == 2) {
833                         return new Tuple3<>(values[0], subValues[0], subValues[1]);
834                     } else {
835                         return new Tuple3<>(values[0], values[1], Constants.MAPPING_TYPE_ARRAY);
836                     }
837                 }
838                 return null;
839             }).collect(Collectors.toMap(Tuple3::getValue1, d -> new Pair<>(d.getValue2(), d.getValue3()))));
840             propMap.put(CRAWLER_METADATA_NAME_MAPPING, params);
841         }
842         return params.get(name);
843     }
844 
845     String getSuggestPopularWordFields();
846 
847     public default String[] getSuggestPopularWordFieldsAsArray() {
848         return split(getSuggestPopularWordFields(), "\n").get(stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
849     }
850 
851     String getSuggestPopularWordTags();
852 
853     public default String[] getSuggestPopularWordTagsAsArray() {
854         return split(getSuggestPopularWordTags(), "\n").get(stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
855     }
856 
857     String getSuggestPopularWordExcludes();
858 
859     public default String[] getSuggestPopularWordExcludesAsArray() {
860         return split(getSuggestPopularWordExcludes(), "\n")
861                 .get(stream -> stream.filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
862     }
863 
864     String getQueryReplaceTermWithPrefixQuery();
865 
866     public default boolean getQueryReplaceTermWithPrefixQueryAsBoolean() {
867         return Boolean.valueOf(getQueryReplaceTermWithPrefixQuery());
868     }
869 
870     String getQueryDefaultLanguages();
871 
872     String getQueryLanguageMapping();
873 
874     public default String[] normalizeQueryLanguages(final String[] langs) {
875         @SuppressWarnings("unchecked")
876         Map<String, String> params = (Map<String, String>) propMap.get(QUERY_LANGUAGE_MAPPING);
877         if (params == null) {
878             params = stream(getQueryLanguageMapping().split("\n")).get(stream -> stream.filter(StringUtil::isNotBlank).map(v -> {
879                 final String[] values = v.split("=");
880                 if (values.length == 2) {
881                     return new Pair<>(values[0], values[1]);
882                 }
883                 return null;
884             }).collect(Collectors.toMap(Pair::getFirst, d -> d.getSecond())));
885             propMap.put(QUERY_LANGUAGE_MAPPING, params);
886         }
887         final Map<String, String> mapping = params;
888         return stream(langs).get(stream -> stream.map(s -> {
889             if (StringUtil.isBlank(s)) {
890                 return null;
891             }
892             final String lang1 = mapping.get(s);
893             if (lang1 != null) {
894                 return lang1;
895             }
896             final String lang2 = mapping.get(s.split("[\\-_]")[0]);
897             if (lang2 != null) {
898                 return lang2;
899             }
900             return null;
901         }).filter(StringUtil::isNotBlank).distinct().toArray(n -> new String[n]));
902     }
903 
904     public default String[] getQueryLanguages(final Enumeration<Locale> locales, final String[] requestLangs) {
905         // requestLangs > default > browser
906         if (StringUtil.isNotBlank(getQueryDefaultLanguages())) {
907             String[] langs = (String[]) propMap.get(QUERY_DEFAULT_LANGUAGES);
908             if (langs == null) {
909                 langs = split(getQueryDefaultLanguages(), ",").get(stream -> stream.map(s -> s.trim()).toArray(n -> new String[n]));
910                 propMap.put(QUERY_DEFAULT_LANGUAGES, langs);
911 
912             }
913             return normalizeQueryLanguages(langs);
914         }
915 
916         if (requestLangs != null && requestLangs.length != 0) {
917             return normalizeQueryLanguages(requestLangs);
918         }
919 
920         if (locales == null) {
921             return StringUtil.EMPTY_STRINGS;
922         }
923 
924         return normalizeQueryLanguages(Collections.list(locales).stream().map(locale -> {
925             final String language = locale.getLanguage();
926             final String country = locale.getCountry();
927             if (StringUtil.isNotBlank(language)) {
928                 if (StringUtil.isNotBlank(country)) {
929                     return language.toLowerCase(Locale.ROOT) + "-" + country.toLowerCase(Locale.ROOT);
930                 }
931                 return language.toLowerCase(Locale.ROOT);
932             }
933             return null;
934         }).toArray(n -> new String[n]));
935     }
936 
937     String getSupportedUploadedFiles();
938 
939     public default boolean isSupportedUploadedFile(final String name) {
940         return split(getSuggestPopularWordExcludes(), ",").get(
941                 stream -> stream.filter(StringUtil::isNotBlank).anyMatch(s -> s.equals(name)));
942     }
943 
944     String getLdapAdminUserObjectClasses();
945 
946     public default Attribute getLdapAdminUserObjectClassAttribute() {
947         final Attribute oc = new BasicAttribute("objectClass");
948         split(getLdapAdminUserObjectClasses(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> oc.add(s.trim())));
949         return oc;
950     }
951 
952     String getLdapAdminUserFilter();
953 
954     public default String getLdapAdminUserFilter(final String name) {
955         return String.format(getLdapAdminUserFilter(), name);
956     }
957 
958     String getLdapAdminUserBaseDn();
959 
960     public default String getLdapAdminUserSecurityPrincipal(final String name) {
961         final StringBuilder buf = new StringBuilder(100);
962         buf.append(String.format(getLdapAdminUserFilter(), name));
963         if (StringUtil.isNotBlank(getLdapAdminUserBaseDn())) {
964             buf.append(',').append(getLdapAdminUserBaseDn());
965         }
966         return buf.toString();
967     }
968 
969     String getLdapAdminRoleObjectClasses();
970 
971     public default Attribute getLdapAdminRoleObjectClassAttribute() {
972         final Attribute oc = new BasicAttribute("objectClass");
973         split(getLdapAdminRoleObjectClasses(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> oc.add(s.trim())));
974         return oc;
975     }
976 
977     String getLdapAdminRoleFilter();
978 
979     public default String getLdapAdminRoleFilter(final String name) {
980         return String.format(getLdapAdminRoleFilter(), name);
981     }
982 
983     String getLdapAdminRoleBaseDn();
984 
985     public default String getLdapAdminRoleSecurityPrincipal(final String name) {
986         final StringBuilder buf = new StringBuilder(100);
987         buf.append(String.format(getLdapAdminRoleFilter(), name));
988         if (StringUtil.isNotBlank(getLdapAdminRoleBaseDn())) {
989             buf.append(',').append(getLdapAdminRoleBaseDn());
990         }
991         return buf.toString();
992     }
993 
994     String getLdapAdminGroupObjectClasses();
995 
996     public default Attribute getLdapAdminGroupObjectClassAttribute() {
997         final Attribute oc = new BasicAttribute("objectClass");
998         split(getLdapAdminGroupObjectClasses(), ",").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> oc.add(s.trim())));
999         return oc;
1000     }
1001 
1002     String getLdapAdminGroupFilter();
1003 
1004     public default String getLdapAdminGroupFilter(final String name) {
1005         return String.format(getLdapAdminGroupFilter(), name);
1006     }
1007 
1008     String getLdapAdminGroupBaseDn();
1009 
1010     public default String getLdapAdminGroupSecurityPrincipal(final String name) {
1011         final StringBuilder buf = new StringBuilder(100);
1012         buf.append(String.format(getLdapAdminGroupFilter(), name));
1013         if (StringUtil.isNotBlank(getLdapAdminGroupBaseDn())) {
1014             buf.append(',').append(getLdapAdminGroupBaseDn());
1015         }
1016         return buf.toString();
1017     }
1018 
1019     String getAuthenticationAdminUsers();
1020 
1021     public default boolean isAdminUser(final String username) {
1022         return split(getAuthenticationAdminUsers(), ",").get(stream -> stream.anyMatch(s -> s.equals(username)));
1023     }
1024 
1025     boolean isLdapAdminEnabled();
1026 
1027     public default boolean isLdapAdminEnabled(final String username) {
1028         if (isAdminUser(username)) {
1029             return false;
1030         }
1031         return isLdapAdminEnabled();
1032     }
1033 
1034     String getCrawlerWebProtocols();
1035 
1036     public default String[] getCrawlerWebProtocolsAsArray() {
1037         return split(getCrawlerWebProtocols(), ",").get(
1038                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim() + ":").toArray(n -> new String[n]));
1039     }
1040 
1041     public default boolean isValidCrawlerWebProtocol(final String url) {
1042         return stream(getCrawlerWebProtocolsAsArray()).get(stream -> stream.anyMatch(s -> url.startsWith(s)));
1043     }
1044 
1045     String getCrawlerFileProtocols();
1046 
1047     public default String[] getCrawlerFileProtocolsAsArray() {
1048         return split(getCrawlerFileProtocols(), ",").get(
1049                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim() + ":").toArray(n -> new String[n]));
1050     }
1051 
1052     public default boolean isValidCrawlerFileProtocol(final String url) {
1053         return stream(getCrawlerFileProtocolsAsArray()).get(stream -> stream.anyMatch(s -> url.startsWith(s)));
1054     }
1055 
1056     public default void processSearchPreference(final SearchRequestBuilder searchRequestBuilder, final OptionalThing<FessUserBean> userBean) {
1057         userBean.map(user -> {
1058             if (user.hasRoles(getAuthenticationAdminRolesAsArray())) {
1059                 return Constants.SEARCH_PREFERENCE_PRIMARY;
1060             }
1061             return user.getUserId();
1062         }).ifPresent(p -> searchRequestBuilder.setPreference(p)).orElse(() -> LaRequestUtil.getOptionalRequest().map(r -> {
1063             final HttpSession session = r.getSession(false);
1064             if (session != null) {
1065                 return session.getId();
1066             }
1067             final String preference = r.getParameter("preference");
1068             if (preference != null) {
1069                 return Integer.toString(preference.hashCode());
1070             }
1071             return null;
1072         }).ifPresent(p -> searchRequestBuilder.setPreference(p)));
1073     }
1074 
1075     String getRoleSearchDefaultPermissions();
1076 
1077     public default String[] getSearchDefaultPermissionsAsArray() {
1078         final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
1079         return split(getRoleSearchDefaultPermissions(), ",")
1080                 .get(stream -> stream.map(p -> permissionHelper.encode(p)).filter(StringUtil::isNotBlank).distinct()
1081                         .toArray(n -> new String[n]));
1082     }
1083 
1084     String getRoleSearchDefaultDisplayPermissions();
1085 
1086     public default String[] getSearchDefaultDisplayEncodedPermissions() {
1087         final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
1088         return split(getRoleSearchDefaultDisplayPermissions(), ",")
1089                 .get(stream -> stream.map(p -> permissionHelper.encode(p)).filter(StringUtil::isNotBlank).distinct()
1090                         .toArray(n -> new String[n]));
1091     }
1092 
1093     public default String getSearchDefaultDisplayPermission() {
1094         return split(getRoleSearchDefaultDisplayPermissions(), ",").get(
1095                 stream -> stream.filter(StringUtil::isNotBlank).distinct().collect(Collectors.joining("\n")));
1096     }
1097 
1098     String getQueryGeoFields();
1099 
1100     public default String[] getQueryGeoFieldsAsArray() {
1101         return split(getQueryGeoFields(), ",").get(
1102                 stream -> stream.map(s -> s.trim()).filter(StringUtil::isNotBlank).toArray(n -> new String[n]));
1103     }
1104 
1105     String getSuggestSearchLogPermissions();
1106 
1107     public default boolean isValidSearchLogPermissions(final String[] permissions) {
1108         if (permissions == null) {
1109             return false;
1110         }
1111         @SuppressWarnings("unchecked")
1112         List<String> validPermissionList = (List<String>) propMap.get(SUGGEST_SEARCH_LOG_PERMISSIONS);
1113         if (validPermissionList == null) {
1114             final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
1115             validPermissionList =
1116                     split(getSuggestSearchLogPermissions(), ",").get(
1117                             stream -> stream.map(s -> permissionHelper.encode(s)).filter(StringUtil::isNotBlank)
1118                                     .collect(Collectors.toList()));
1119             propMap.put(SUGGEST_SEARCH_LOG_PERMISSIONS, validPermissionList);
1120         }
1121         final List<String> list = validPermissionList;
1122         return stream(permissions).get(stream -> stream.allMatch(v -> list.contains(v)));
1123     }
1124 
1125     String getRoleSearchUserPrefix();
1126 
1127     String getRoleSearchGuestPermissions();
1128 
1129     public default List<String> getSearchGuestPermissionList() {
1130         @SuppressWarnings("unchecked")
1131         List<String> list = (List<String>) propMap.get(SEARCH_GUEST_PERMISSION_LIST);
1132         if (list == null) {
1133             final PermissionHelper permissionHelper = ComponentUtil.getPermissionHelper();
1134             list =
1135                     split(getRoleSearchGuestPermissions(), ",").get(
1136                             stream -> stream.map(s -> permissionHelper.encode(s)).filter(StringUtil::isNotBlank)
1137                                     .collect(Collectors.toList()));
1138             list.add(getRoleSearchUserPrefix() + Constants.GUEST_USER);
1139             propMap.put(SEARCH_GUEST_PERMISSION_LIST, list);
1140         }
1141         return list;
1142     }
1143 
1144     String getOicDefaultGroups();
1145 
1146     public default String[] getOicDefaultGroupsAsArray() {
1147         String[] array = (String[]) propMap.get(OIC_DEFAULT_GROUPS);
1148         if (array == null) {
1149             if (StringUtil.isBlank(getOicDefaultGroups())) {
1150                 array = StringUtil.EMPTY_STRINGS;
1151             } else {
1152                 array =
1153                         split(getOicDefaultGroups(), ",").get(
1154                                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).toArray(n -> new String[n]));
1155             }
1156             propMap.put(OIC_DEFAULT_GROUPS, array);
1157         }
1158         return array;
1159     }
1160 
1161     String getOicDefaultRoles();
1162 
1163     public default String[] getOicDefaultRolesAsArray() {
1164         String[] array = (String[]) propMap.get(OIC_DEFAULT_ROLES);
1165         if (array == null) {
1166             if (StringUtil.isBlank(getOicDefaultRoles())) {
1167                 array = StringUtil.EMPTY_STRINGS;
1168             } else {
1169                 array =
1170                         split(getOicDefaultRoles(), ",").get(
1171                                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).toArray(n -> new String[n]));
1172             }
1173             propMap.put(OIC_DEFAULT_ROLES, array);
1174         }
1175         return array;
1176     }
1177 
1178     String getIndexAdminArrayFields();
1179 
1180     public default Set<String> getIndexAdminArrayFieldSet() {
1181         @SuppressWarnings("unchecked")
1182         Set<String> fieldSet = (Set<String>) propMap.get(INDEX_ADMIN_ARRAY_FIELD_SET);
1183         if (fieldSet == null) {
1184             fieldSet =
1185                     split(getIndexAdminArrayFields(), ",").get(
1186                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1187             propMap.put(INDEX_ADMIN_ARRAY_FIELD_SET, fieldSet);
1188         }
1189         return fieldSet;
1190     }
1191 
1192     public default boolean validateIndexArrayFields(final Map<String, Object> source) {
1193         return invalidIndexArrayFields(source).isEmpty();
1194     }
1195 
1196     public default List<String> invalidIndexArrayFields(final Map<String, Object> source) {
1197         // TODO always returns empty list
1198         return split(getIndexAdminArrayFields(), ",").get(
1199                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).filter(s -> isNonEmptyValue(source.get(s)))
1200                         .filter(s -> false) // TODO
1201                         .collect(Collectors.toList()));
1202     }
1203 
1204     String getIndexAdminDateFields();
1205 
1206     public default Set<String> getIndexAdminDateFieldSet() {
1207         @SuppressWarnings("unchecked")
1208         Set<String> fieldSet = (Set<String>) propMap.get(INDEX_ADMIN_DATE_FIELD_SET);
1209         if (fieldSet == null) {
1210             fieldSet =
1211                     split(getIndexAdminDateFields(), ",").get(
1212                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1213             propMap.put(INDEX_ADMIN_DATE_FIELD_SET, fieldSet);
1214         }
1215         return fieldSet;
1216     }
1217 
1218     public default boolean validateIndexDateFields(final Map<String, Object> source) {
1219         return invalidIndexDateFields(source).isEmpty();
1220     }
1221 
1222     public default List<String> invalidIndexDateFields(final Map<String, Object> source) {
1223         return split(getIndexAdminDateFields(), ",").get(
1224                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).filter(s -> isNonEmptyValue(source.get(s)))
1225                         .filter(s -> !validateDateTimeString(source.get(s))).collect(Collectors.toList()));
1226     }
1227 
1228     public default boolean validateDateTimeString(final Object obj) {
1229         if (FessFunctions.parseDate(obj.toString()) != null) {
1230             return true;
1231         }
1232         return false;
1233     }
1234 
1235     String getIndexAdminIntegerFields();
1236 
1237     public default Set<String> getIndexAdminIntegerFieldSet() {
1238         @SuppressWarnings("unchecked")
1239         Set<String> fieldSet = (Set<String>) propMap.get(INDEX_ADMIN_INTEGER_FIELD_SET);
1240         if (fieldSet == null) {
1241             fieldSet =
1242                     split(getIndexAdminIntegerFields(), ",").get(
1243                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1244             propMap.put(INDEX_ADMIN_INTEGER_FIELD_SET, fieldSet);
1245         }
1246         return fieldSet;
1247     }
1248 
1249     public default boolean validateIndexIntegerFields(final Map<String, Object> source) {
1250         return invalidIndexIntegerFields(source).isEmpty();
1251     }
1252 
1253     public default List<String> invalidIndexIntegerFields(final Map<String, Object> source) {
1254         final IntegerTypeValidator integerValidator = new IntegerTypeValidator();
1255         return split(getIndexAdminIntegerFields(), ",").get(
1256                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).filter(s -> isNonEmptyValue(source.get(s)))
1257                         .filter(s -> !integerValidator.isValid(source.get(s).toString(), null)).collect(Collectors.toList()));
1258     }
1259 
1260     String getIndexAdminLongFields();
1261 
1262     public default Set<String> getIndexAdminLongFieldSet() {
1263         @SuppressWarnings("unchecked")
1264         Set<String> fieldSet = (Set<String>) propMap.get(INDEX_ADMIN_LONG_FIELD_SET);
1265         if (fieldSet == null) {
1266             fieldSet =
1267                     split(getIndexAdminLongFields(), ",").get(
1268                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1269             propMap.put(INDEX_ADMIN_LONG_FIELD_SET, fieldSet);
1270         }
1271         return fieldSet;
1272     }
1273 
1274     public default boolean validateIndexLongFields(final Map<String, Object> source) {
1275         return invalidIndexLongFields(source).isEmpty();
1276     }
1277 
1278     public default List<String> invalidIndexLongFields(final Map<String, Object> source) {
1279         final LongTypeValidator longValidator = new LongTypeValidator();
1280         return split(getIndexAdminLongFields(), ",").get(
1281                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).filter(s -> isNonEmptyValue(source.get(s)))
1282                         .filter(s -> !longValidator.isValid(source.get(s).toString(), null)).collect(Collectors.toList()));
1283     }
1284 
1285     String getIndexAdminFloatFields();
1286 
1287     public default Set<String> getIndexAdminFloatFieldSet() {
1288         @SuppressWarnings("unchecked")
1289         Set<String> fieldSet = (Set<String>) propMap.get(INDEX_ADMIN_FLOAT_FIELD_SET);
1290         if (fieldSet == null) {
1291             fieldSet =
1292                     split(getIndexAdminFloatFields(), ",").get(
1293                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1294             propMap.put(INDEX_ADMIN_FLOAT_FIELD_SET, fieldSet);
1295         }
1296         return fieldSet;
1297     }
1298 
1299     public default boolean validateIndexFloatFields(final Map<String, Object> source) {
1300         return invalidIndexFloatFields(source).isEmpty();
1301     }
1302 
1303     public default List<String> invalidIndexFloatFields(final Map<String, Object> source) {
1304         final FloatTypeValidator floatValidator = new FloatTypeValidator();
1305         return split(getIndexAdminFloatFields(), ",").get(
1306                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).filter(s -> isNonEmptyValue(source.get(s)))
1307                         .filter(s -> !floatValidator.isValid(source.get(s).toString(), null)).collect(Collectors.toList()));
1308     }
1309 
1310     String getIndexAdminDoubleFields();
1311 
1312     public default Set<String> getIndexAdminDoubleFieldSet() {
1313         @SuppressWarnings("unchecked")
1314         Set<String> fieldSet = (Set<String>) propMap.get(INDEX_ADMIN_DOUBLE_FIELD_SET);
1315         if (fieldSet == null) {
1316             fieldSet =
1317                     split(getIndexAdminDoubleFields(), ",").get(
1318                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1319             propMap.put(INDEX_ADMIN_DOUBLE_FIELD_SET, fieldSet);
1320         }
1321         return fieldSet;
1322     }
1323 
1324     public default boolean validateIndexDoubleFields(final Map<String, Object> source) {
1325         return invalidIndexDoubleFields(source).isEmpty();
1326     }
1327 
1328     public default List<String> invalidIndexDoubleFields(final Map<String, Object> source) {
1329         final DoubleTypeValidator doubleValidator = new DoubleTypeValidator();
1330         return split(getIndexAdminDoubleFields(), ",").get(
1331                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).filter(s -> isNonEmptyValue(source.get(s)))
1332                         .filter(s -> !doubleValidator.isValid(source.get(s).toString(), null)).collect(Collectors.toList()));
1333     }
1334 
1335     public default Map<String, Object> convertToEditableDoc(final Map<String, Object> source) {
1336 
1337         final Set<String> arrayFieldSet = getIndexAdminArrayFieldSet();
1338         final Set<String> dateFieldSet = getIndexAdminDateFieldSet();
1339         final Set<String> integerFieldSet = getIndexAdminIntegerFieldSet();
1340         final Set<String> longFieldSet = getIndexAdminLongFieldSet();
1341         final Set<String> floatFieldSet = getIndexAdminFloatFieldSet();
1342         final Set<String> doubleFieldSet = getIndexAdminDoubleFieldSet();
1343 
1344         return source.entrySet().stream().map(e -> {
1345             final String key = e.getKey();
1346             Object value = e.getValue();
1347             if (value == null) {
1348                 value = StringUtil.EMPTY;
1349             }
1350             if (value instanceof String || value == null) {
1351                 return new Pair<>(key, value);
1352             }
1353             if (arrayFieldSet.contains(key)) {
1354                 if (value instanceof String[]) {
1355                     value = stream((String[]) value).get(stream -> stream.collect(Collectors.joining("\n")));
1356                 } else if (value instanceof List) {
1357                     @SuppressWarnings("unchecked")
1358                     final List<String> list = (List<String>) value;
1359                     value = list.stream().collect(Collectors.joining("\n"));
1360                 }
1361             } else if (dateFieldSet.contains(key)) {
1362                 value = FessFunctions.formatDate((Date) value);
1363             } else if (integerFieldSet.contains(key)) {
1364                 value = value.toString();
1365             } else if (longFieldSet.contains(key)) {
1366                 value = value.toString();
1367             } else if (floatFieldSet.contains(key)) {
1368                 value = value.toString();
1369             } else if (doubleFieldSet.contains(key)) {
1370                 value = value.toString();
1371             }
1372             return new Pair<>(key, value);
1373         }).collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
1374     }
1375 
1376     public default Map<String, Object> convertToStorableDoc(final Map<String, Object> source) {
1377 
1378         final Set<String> arrayFieldSet = getIndexAdminArrayFieldSet();
1379         final Set<String> dateFieldSet = getIndexAdminDateFieldSet();
1380         final Set<String> integerFieldSet = getIndexAdminIntegerFieldSet();
1381         final Set<String> longFieldSet = getIndexAdminLongFieldSet();
1382         final Set<String> floatFieldSet = getIndexAdminFloatFieldSet();
1383         final Set<String> doubleFieldSet = getIndexAdminDoubleFieldSet();
1384 
1385         return source
1386                 .entrySet()
1387                 .stream()
1388                 .filter(e -> isNonEmptyValue(e.getValue()))
1389                 .map(e -> {
1390                     final String key = e.getKey();
1391                     Object value = e.getValue();
1392                     if (arrayFieldSet.contains(key)) {
1393                         value =
1394                                 split(value.toString(), "\n").get(
1395                                         stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toList()));
1396                     } else if (dateFieldSet.contains(key)) {
1397                         // TODO time zone
1398                         value = FessFunctions.parseDate(value.toString());
1399                     } else if (integerFieldSet.contains(key)) {
1400                         value = Integer.parseInt(value.toString());
1401                     } else if (longFieldSet.contains(key)) {
1402                         value = Long.parseLong(value.toString());
1403                     } else if (floatFieldSet.contains(key)) {
1404                         value = Float.parseFloat(value.toString());
1405                     } else if (doubleFieldSet.contains(key)) {
1406                         value = Double.parseDouble(value.toString());
1407                     }
1408                     return new Pair<>(key, value);
1409                 }).collect(Collectors.toMap(Pair::getFirst, Pair::getSecond));
1410     }
1411 
1412     String getIndexAdminRequiredFields();
1413 
1414     public default boolean validateIndexRequiredFields(final Map<String, Object> source) {
1415         return invalidIndexRequiredFields(source).isEmpty();
1416     }
1417 
1418     public default List<String> invalidIndexRequiredFields(final Map<String, Object> source) {
1419         final RequiredValidator requiredValidator = new RequiredValidator();
1420         return split(getIndexAdminRequiredFields(), ",").get(
1421                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim())
1422                         .filter(s -> !requiredValidator.isValid(source.get(s), null)).collect(Collectors.toList()));
1423     }
1424 
1425     public default boolean isNonEmptyValue(final Object value) {
1426         final RequiredValidator requiredValidator = new RequiredValidator();
1427         return requiredValidator.isValid(value, null);
1428     }
1429 
1430     String getCrawlerDocumentSpaceChars();
1431 
1432     public default int[] getCrawlerDocumentSpaceCharsAsArray() {
1433         int[] spaceChars = (int[]) propMap.get(CRAWLER_DOCUMENT_SPACE_CHARS);
1434         if (spaceChars == null) {
1435             final String spaceStr = getCrawlerDocumentSpaceChars();
1436             if (spaceStr.startsWith("u")) {
1437                 spaceChars =
1438                         split(spaceStr, "u").get(
1439                                 stream -> stream.filter(StringUtil::isNotBlank).mapToInt(s -> Integer.parseInt(s, 16)).toArray());
1440             } else {
1441                 // backward compatibility
1442                 final int length = spaceStr.length();
1443                 spaceChars = new int[length];
1444                 for (int i = 0; i < length; i++) {
1445                     spaceChars[i] = spaceStr.codePointAt(i);
1446                 }
1447             }
1448             propMap.put(CRAWLER_DOCUMENT_SPACE_CHARS, spaceChars);
1449         }
1450         return spaceChars;
1451     }
1452 
1453     String getQueryAdditionalResponseFields();
1454 
1455     public default String[] getQueryAdditionalResponseFields(final String... fields) {
1456         final List<String> list = new ArrayList<>(fields.length + 10);
1457         stream(fields).of(stream -> stream.forEach(list::add));
1458         split(getQueryAdditionalResponseFields(), ",").of(
1459                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1460         return list.toArray(new String[list.size()]);
1461     }
1462 
1463     String getQueryAdditionalCacheResponseFields();
1464 
1465     public default String[] getQueryAdditionalCacheResponseFields(final String... fields) {
1466         final List<String> list = new ArrayList<>(fields.length + 10);
1467         stream(fields).of(stream -> stream.forEach(list::add));
1468         split(getQueryAdditionalCacheResponseFields(), ",").of(
1469                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1470         return list.toArray(new String[list.size()]);
1471     }
1472 
1473     String getQueryAdditionalHighlightedFields();
1474 
1475     public default String[] getQueryAdditionalHighlightedFields(final String... fields) {
1476         final List<String> list = new ArrayList<>(fields.length + 10);
1477         stream(fields).of(stream -> stream.forEach(list::add));
1478         split(getQueryAdditionalHighlightedFields(), ",").of(
1479                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1480         return list.toArray(new String[list.size()]);
1481     }
1482 
1483     String getQueryAdditionalSearchFields();
1484 
1485     public default String[] getQueryAdditionalSearchFields(final String... fields) {
1486         final List<String> list = new ArrayList<>(fields.length + 10);
1487         stream(fields).of(stream -> stream.forEach(list::add));
1488         split(getQueryAdditionalSearchFields(), ",").of(
1489                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1490         return list.toArray(new String[list.size()]);
1491     }
1492 
1493     String getQueryAdditionalFacetFields();
1494 
1495     public default String[] getQueryAdditionalFacetFields(final String... fields) {
1496         final List<String> list = new ArrayList<>(fields.length + 10);
1497         stream(fields).of(stream -> stream.forEach(list::add));
1498         split(getQueryAdditionalFacetFields(), ",").of(
1499                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1500         return list.toArray(new String[list.size()]);
1501     }
1502 
1503     String getQueryAdditionalSortFields();
1504 
1505     public default String[] getQueryAdditionalSortFields(final String... fields) {
1506         final List<String> list = new ArrayList<>(fields.length + 10);
1507         stream(fields).of(stream -> stream.forEach(list::add));
1508         split(getQueryAdditionalSortFields(), ",")
1509                 .of(stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1510         return list.toArray(new String[list.size()]);
1511     }
1512 
1513     String getQueryAdditionalApiResponseFields();
1514 
1515     public default String[] getQueryAdditionalApiResponseFields(final String... fields) {
1516         final List<String> list = new ArrayList<>(fields.length + 10);
1517         stream(fields).of(stream -> stream.forEach(list::add));
1518         split(getQueryAdditionalApiResponseFields(), ",").of(
1519                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1520         return list.toArray(new String[list.size()]);
1521     }
1522 
1523     String getQueryAdditionalNotAnalyzedFields();
1524 
1525     public default String[] getQueryAdditionalNotAnalyzedFields(final String... fields) {
1526         final List<String> list = new ArrayList<>(fields.length + 10);
1527         stream(fields).of(stream -> stream.forEach(list::add));
1528         split(getQueryAdditionalNotAnalyzedFields(), ",").of(
1529                 stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).forEach(list::add));
1530         return list.toArray(new String[list.size()]);
1531     }
1532 
1533     String getThumbnailGeneratorTargets();
1534 
1535     public default String[] getThumbnailGeneratorTargetsAsArray() {
1536         return getThumbnailGeneratorTargets().split(",");
1537 
1538     }
1539 
1540     String getApiAdminAccessPermissions();
1541 
1542     public default Set<String> getApiAdminAccessPermissionSet() {
1543         @SuppressWarnings("unchecked")
1544         Set<String> fieldSet = (Set<String>) propMap.get(API_ADMIN_ACCESS_PERMISSION_SET);
1545         if (fieldSet == null) {
1546             fieldSet =
1547                     split(getApiAdminAccessPermissions(), ",").get(
1548                             stream -> stream.filter(StringUtil::isNotBlank).map(s -> s.trim()).collect(Collectors.toSet()));
1549             propMap.put(API_ADMIN_ACCESS_PERMISSION_SET, fieldSet);
1550         }
1551         return fieldSet;
1552     }
1553 
1554     public default boolean isApiAdminAccessAllowed(final Set<String> accessPermissions) {
1555         return getApiAdminAccessPermissionSet().stream().anyMatch(s -> accessPermissions.contains(s));
1556     }
1557 
1558     String getUserCodePattern();
1559 
1560     public default boolean isValidUserCode(final String userCode) {
1561         if (userCode == null) {
1562             return false;
1563         }
1564         Pattern pattern = (Pattern) propMap.get(USER_CODE_PATTERN);
1565         if (pattern == null) {
1566             pattern = Pattern.compile(getUserCodePattern());
1567             propMap.put(USER_CODE_PATTERN, pattern);
1568         }
1569         return pattern.matcher(userCode).matches();
1570     }
1571 
1572     String getQueryCollapseInnerHitsSorts();
1573 
1574     @SuppressWarnings("rawtypes")
1575     public default OptionalThing<SortBuilder[]> getQueryCollapseInnerHitsSortBuilders() {
1576         @SuppressWarnings("unchecked")
1577         OptionalThing<SortBuilder[]> ot = (OptionalThing<SortBuilder[]>) propMap.get(QUERY_COLLAPSE_INNER_HITS_SORTS);
1578         if (ot == null) {
1579             final String sorts = getQueryCollapseInnerHitsSorts();
1580             if (StringUtil.isBlank(sorts)) {
1581                 ot = OptionalThing.empty();
1582             } else {
1583                 final SortBuilder[] sortBuilders =
1584                         split(sorts, ",").get(
1585                                 stream -> stream
1586                                         .filter(StringUtil::isNotBlank)
1587                                         .map(s -> {
1588                                             final String[] values = s.split(":");
1589                                             if (values.length > 1) {
1590                                                 return SortBuilders.fieldSort(values[0]).order(
1591                                                         values[0].equalsIgnoreCase("desc") ? SortOrder.DESC : SortOrder.ASC);
1592                                             } else {
1593                                                 return SortBuilders.fieldSort(values[0]).order(SortOrder.ASC);
1594                                             }
1595                                         }).toArray(n -> new SortBuilder[n]));
1596                 ot = OptionalThing.of(sortBuilders);
1597             }
1598             propMap.put(QUERY_COLLAPSE_INNER_HITS_SORTS, ot);
1599         }
1600         return ot;
1601     }
1602 
1603     String getVirtualHostHeaders();
1604 
1605     public default HtmlNext getVirtualHostPath(final HtmlNext page) {
1606         return processVirtualHost(s -> {
1607             final String basePath = StringUtil.isBlank(s) ? StringUtil.EMPTY : "/" + s;
1608             return new HtmlNext(basePath + page.getRoutingPath());
1609         }, page);
1610     }
1611 
1612     public default String getVirtualHostKey() {
1613         return LaRequestUtil.getOptionalRequest().map(req -> (String) req.getAttribute(VIRTUAL_HOST_VALUE)).orElseGet(() -> {
1614             final String value = processVirtualHost(s -> s, StringUtil.EMPTY);
1615             LaRequestUtil.getOptionalRequest().ifPresent(req -> req.setAttribute(VIRTUAL_HOST_VALUE, value));
1616             return value;
1617         });
1618     }
1619 
1620     public default String getVirtualHostHeaderValue() {
1621         final String value = getVirtualHostValue();
1622         if (StringUtil.isNotBlank(value)) {
1623             return value;
1624         }
1625         return getVirtualHostHeaders();
1626     }
1627 
1628     public default <T> T processVirtualHost(final Function<String, T> func, final T defaultValue) {
1629         final Tuple3<String, String, String>[] vHosts = getVirtualHosts();
1630         return LaRequestUtil.getOptionalRequest().map(req -> {
1631             for (final Tuple3<String, String, String> host : vHosts) {
1632                 final String headerValue = req.getHeader(host.getValue1());
1633                 if (host.getValue2().equalsIgnoreCase(headerValue)) {
1634                     return func.apply(host.getValue3());
1635                 }
1636             }
1637             return defaultValue;
1638         }).orElse(defaultValue);
1639     }
1640 
1641     public default String[] getVirtualHostKeys() {
1642         return stream(getVirtualHosts()).get(stream -> stream.map(h -> h.getValue3()).toArray(n -> new String[n]));
1643     }
1644 
1645     public default String[] getVirtualHostPaths() {
1646         return stream(getVirtualHosts()).get(stream -> stream.map(h -> "/" + h.getValue3()).toArray(n -> new String[n]));
1647     }
1648 
1649     @SuppressWarnings("unchecked")
1650     public default Tuple3<String, String, String>[] getVirtualHosts() {
1651         Tuple3<String, String, String>[] hosts = (Tuple3<String, String, String>[]) propMap.get(VIRTUAL_HOST_HEADERS);
1652         if (hosts == null) {
1653             hosts =
1654                     split(getVirtualHostHeaderValue(), "\n").get(
1655                             stream -> stream
1656                                     .map(s -> {
1657                                         final String[] v1 = s.split("=");
1658                                         if (v1.length == 2) {
1659                                             final String[] v2 = v1[0].split(":", 2);
1660                                             if (v2.length == 2) {
1661                                                 return new Tuple3<>(v2[0].trim(), v2[1].trim(), v1[1].replaceAll("[^a-zA-Z0-9_]",
1662                                                         StringUtil.EMPTY).trim());
1663                                             }
1664                                         }
1665                                         return null;
1666                                     })
1667                                     .filter(v -> {
1668                                         if (v == null) {
1669                                             return false;
1670                                         }
1671                                         if ("admin".equalsIgnoreCase(v.getValue3()) || "common".equalsIgnoreCase(v.getValue3())
1672                                                 || "error".equalsIgnoreCase(v.getValue3()) || "login".equalsIgnoreCase(v.getValue3())
1673                                                 || "profile".equalsIgnoreCase(v.getValue3())) {
1674                                             return false;
1675                                         }
1676                                         return true;
1677                                     }).toArray(n -> new Tuple3[n]));
1678             propMap.put(VIRTUAL_HOST_HEADERS, hosts);
1679         }
1680         return hosts;
1681     }
1682 
1683     String getCrawlerFailureUrlStatusCodes();
1684 
1685     public default boolean isCrawlerFailureUrlStatusCodes(final int code) {
1686         int[] codes = (int[]) propMap.get(CRAWLER_FAILURE_URL_STATUS_CODES);
1687         if (codes == null) {
1688             codes =
1689                     split(getCrawlerFailureUrlStatusCodes(), ",").get(
1690                             stream -> stream.filter(StringUtil::isNotBlank).mapToInt(Integer::parseInt).toArray());
1691             propMap.put(CRAWLER_FAILURE_URL_STATUS_CODES, codes);
1692         }
1693         for (final int v : codes) {
1694             if (v == code) {
1695                 return true;
1696             }
1697         }
1698         return false;
1699     }
1700 
1701     Integer getThumbnailHtmlImageMinWidthAsInteger();
1702 
1703     Integer getThumbnailHtmlImageMinHeightAsInteger();
1704 
1705     java.math.BigDecimal getThumbnailHtmlImageMaxAspectRatioAsDecimal();
1706 
1707     public default boolean validateThumbnailSize(final int width, final int height) {
1708         if (width <= 0 || height <= 0) {
1709             return false;
1710         }
1711 
1712         if (width < getThumbnailHtmlImageMinWidthAsInteger().intValue() || height < getThumbnailHtmlImageMinHeightAsInteger().intValue()) {
1713             return false;
1714         }
1715 
1716         final float ratio = getThumbnailHtmlImageMaxAspectRatioAsDecimal().floatValue();
1717         if (((float) width) / ((float) height) > ratio || ((float) height) / ((float) width) > ratio) {
1718             return false;
1719         }
1720 
1721         return true;
1722     }
1723 
1724     String getHttpProxyHost();
1725 
1726     Integer getHttpProxyPortAsInteger();
1727 
1728     String getHttpProxyUsername();
1729 
1730     String getHttpProxyPassword();
1731 
1732     public default Proxy getHttpProxy() {
1733         Proxy proxy = (Proxy) propMap.get(HTML_PROXY);
1734         if (proxy == null) {
1735             if (StringUtil.isNotBlank(getHttpProxyHost()) && getHttpProxyPortAsInteger() != null) {
1736                 final SocketAddress addr = new InetSocketAddress(getHttpProxyHost(), getHttpProxyPortAsInteger());
1737                 proxy = new Proxy(Type.HTTP, addr);
1738                 if (StringUtil.isNotBlank(getHttpProxyUsername())) {
1739                     Authenticator.setDefault(new Authenticator() {
1740                         @Override
1741                         protected PasswordAuthentication getPasswordAuthentication() {
1742                             return new PasswordAuthentication(getHttpProxyUsername(), getHttpProxyPassword().toCharArray());
1743                         }
1744                     });
1745                 }
1746             } else {
1747                 proxy = Proxy.NO_PROXY;
1748             }
1749             propMap.put(HTML_PROXY, proxy);
1750         }
1751         return proxy;
1752     }
1753 
1754     String getThumbnailHtmlImageExcludeExtensions();
1755 
1756     public default boolean isThumbnailHtmlImageUrl(final String url) {
1757         if (StringUtil.isBlank(url)) {
1758             return false;
1759         }
1760 
1761         String[] excludeExtensions = (String[]) propMap.get(THUMBNAIL_HTML_IMAGE_EXCLUDE_EXTENSIONS);
1762         if (excludeExtensions == null) {
1763             excludeExtensions =
1764                     split(getThumbnailHtmlImageExcludeExtensions(), ",").get(
1765                             stream -> stream.map(s -> s.toLowerCase(Locale.ROOT).trim()).filter(StringUtil::isNotBlank)
1766                                     .toArray(n -> new String[n]));
1767             propMap.put(THUMBNAIL_HTML_IMAGE_EXCLUDE_EXTENSIONS, excludeExtensions);
1768         }
1769 
1770         final String u = url.toLowerCase(Locale.ROOT);
1771         return !stream(excludeExtensions).get(stream -> stream.anyMatch(s -> u.endsWith(s)));
1772     }
1773 
1774 }