View Javadoc
1   /*
2    * Copyright 2012-2021 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.thumbnail.impl;
17  
18  import static org.codelibs.core.stream.StreamUtil.stream;
19  
20  import java.io.File;
21  import java.util.ArrayList;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.function.BiPredicate;
26  import java.util.function.Predicate;
27  
28  import org.apache.logging.log4j.LogManager;
29  import org.apache.logging.log4j.Logger;
30  import org.codelibs.core.lang.StringUtil;
31  import org.codelibs.core.misc.Tuple3;
32  import org.codelibs.fess.crawler.builder.RequestDataBuilder;
33  import org.codelibs.fess.crawler.client.CrawlerClient;
34  import org.codelibs.fess.crawler.client.CrawlerClientFactory;
35  import org.codelibs.fess.crawler.entity.ResponseData;
36  import org.codelibs.fess.crawler.exception.CrawlingAccessException;
37  import org.codelibs.fess.es.client.SearchEngineClient;
38  import org.codelibs.fess.es.config.exentity.CrawlingConfig;
39  import org.codelibs.fess.exception.ThumbnailGenerationException;
40  import org.codelibs.fess.helper.CrawlingConfigHelper;
41  import org.codelibs.fess.helper.IndexingHelper;
42  import org.codelibs.fess.mylasta.direction.FessConfig;
43  import org.codelibs.fess.thumbnail.ThumbnailGenerator;
44  import org.codelibs.fess.util.ComponentUtil;
45  import org.codelibs.fess.util.DocumentUtil;
46  
47  public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
48      private static final Logger logger = LogManager.getLogger(BaseThumbnailGenerator.class);
49  
50      protected final Map<String, String> conditionMap = new HashMap<>();
51  
52      protected int directoryNameLength = 5;
53  
54      protected List<String> generatorList;
55  
56      protected Map<String, String> filePathMap = new HashMap<>();
57  
58      protected String name;
59  
60      protected int maxRedirectCount = 10;
61  
62      protected Boolean available = null;
63  
64      public void register() {
65          ComponentUtil.getThumbnailManager().add(this);
66      }
67  
68      public void addCondition(final String key, final String regex) {
69          final String value = conditionMap.get(key);
70          if (StringUtil.isBlank(value)) {
71              conditionMap.put(key, regex);
72          } else {
73              conditionMap.put(key, value + "|" + regex);
74          }
75      }
76  
77      @Override
78      public boolean isTarget(final Map<String, Object> docMap) {
79          if (!docMap.containsKey(ComponentUtil.getFessConfig().getIndexFieldThumbnail())) {
80              return false;
81          }
82          for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
83              final Object value = docMap.get(entry.getKey());
84              if (value instanceof String && ((String) value).matches(entry.getValue())) {
85                  return true;
86              }
87          }
88          return false;
89      }
90  
91      @Override
92      public boolean isAvailable() {
93          if (available != null) {
94              return available;
95          }
96          if (generatorList != null && !generatorList.isEmpty()) {
97              String path = System.getenv("PATH");
98              if (path == null) {
99                  path = System.getenv("Path");
100             }
101             if (path == null) {
102                 path = System.getenv("path");
103             }
104             final List<String> pathList = new ArrayList<>();
105             pathList.add("/usr/share/fess/bin");
106             if (path != null) {
107                 stream(path.split(File.pathSeparator)).of(stream -> stream.map(String::trim).forEach(s -> pathList.add(s)));
108             }
109             available = generatorList.stream().map(s -> {
110                 if (s.startsWith("${path}")) {
111                     for (final String p : pathList) {
112                         final File f = new File(s.replace("${path}", p));
113                         if (f.exists()) {
114                             final String filePath = f.getAbsolutePath();
115                             filePathMap.put(s, filePath);
116                             return filePath;
117                         }
118                     }
119                 }
120                 return s;
121             }).allMatch(s -> new File(s).isFile());
122         } else {
123             available = true;
124         }
125         return available;
126     }
127 
128     @Override
129     public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
130         final FessConfig fessConfig = ComponentUtil.getFessConfig();
131         final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
132         final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
133         if (logger.isDebugEnabled()) {
134             logger.debug("Create thumbnail task: {}", task);
135         }
136         return task;
137     }
138 
139     public void setDirectoryNameLength(final int directoryNameLength) {
140         this.directoryNameLength = directoryNameLength;
141     }
142 
143     protected String expandPath(final String value) {
144         if (value != null && filePathMap.containsKey(value)) {
145             return filePathMap.get(value);
146         }
147         return value;
148     }
149 
150     protected void updateThumbnailField(final String thumbnailId, final String value) {
151         // TODO bulk
152         final FessConfig fessConfig = ComponentUtil.getFessConfig();
153         try {
154             ComponentUtil.getIndexingHelper().updateDocument(ComponentUtil.getSearchEngineClient(), thumbnailId,
155                     fessConfig.getIndexFieldThumbnail(), value);
156         } catch (final Exception e) {
157             logger.warn("Failed to update thumbnail field at {}", thumbnailId, e);
158         }
159     }
160 
161     protected boolean process(final String id, final BiPredicate<String, String> consumer) {
162         final FessConfig fessConfig = ComponentUtil.getFessConfig();
163         final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
164         final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
165         try {
166             final Map<String, Object> doc = indexingHelper.getDocument(searchEngineClient, id,
167                     new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
168             if (doc == null) {
169                 throw new ThumbnailGenerationException("Document is not found: " + id);
170             }
171             final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
172             if (StringUtil.isBlank(url)) {
173                 throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
174             }
175             final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
176             if (configId == null || configId.length() < 2) {
177                 throw new ThumbnailGenerationException("Invalid configId: " + configId);
178             }
179             return consumer.test(configId, url);
180         } catch (final ThumbnailGenerationException e) {
181             if (e.getCause() == null) {
182                 logger.debug(e.getMessage());
183             } else {
184                 logger.warn("Failed to process {}", id, e);
185             }
186         } catch (final Exception e) {
187             logger.warn("Failed to process {}", id, e);
188         }
189         return false;
190     }
191 
192     protected boolean process(final String id, final Predicate<ResponseData> consumer) {
193         return process(id, (configId, url) -> {
194             final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
195             final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
196             if (config == null) {
197                 throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
198             }
199 
200             if (logger.isInfoEnabled()) {
201                 logger.info("Generating Thumbnail: {}", url);
202             }
203 
204             final CrawlerClientFactory crawlerClientFactory =
205                     config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
206             final CrawlerClient client = crawlerClientFactory.getClient(url);
207             if (client == null) {
208                 throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
209             }
210             String u = url;
211             for (int i = 0; i < maxRedirectCount; i++) {
212                 try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
213                     if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
214                         u = responseData.getRedirectLocation();
215                         continue;
216                     }
217                     if (StringUtil.isBlank(responseData.getUrl())) {
218                         throw new ThumbnailGenerationException(
219                                 "Failed to process a thumbnail content: " + url + " (Response URL is empty)");
220                     }
221                     return consumer.test(responseData);
222                 } catch (final CrawlingAccessException e) {
223                     if (logger.isDebugEnabled()) {
224                         throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
225                     }
226                     throw new ThumbnailGenerationException(e.getMessage());
227                 } catch (final Exception e) {
228                     throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
229                 }
230             }
231             throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
232         });
233     }
234 
235     public void setGeneratorList(final List<String> generatorList) {
236         this.generatorList = generatorList;
237     }
238 
239     @Override
240     public String getName() {
241         return name;
242     }
243 
244     public void setName(final String name) {
245         this.name = name;
246     }
247 
248     public void setMaxRedirectCount(final int maxRedirectCount) {
249         this.maxRedirectCount = maxRedirectCount;
250     }
251 
252 }