View Javadoc
1   /*
2    * Copyright 2012-2025 CodeLibs Project and the Others.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    *     http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
13   * either express or implied. See the License for the specific language
14   * governing permissions and limitations under the License.
15   */
16  package org.codelibs.fess.thumbnail.impl;
17  
18  import static org.codelibs.core.stream.StreamUtil.stream;
19  
20  import java.io.File;
21  import java.util.ArrayList;
22  import java.util.HashMap;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.function.BiPredicate;
26  import java.util.function.Predicate;
27  
28  import org.apache.logging.log4j.LogManager;
29  import org.apache.logging.log4j.Logger;
30  import org.codelibs.core.lang.StringUtil;
31  import org.codelibs.core.misc.Tuple3;
32  import org.codelibs.fess.crawler.builder.RequestDataBuilder;
33  import org.codelibs.fess.crawler.client.CrawlerClient;
34  import org.codelibs.fess.crawler.client.CrawlerClientFactory;
35  import org.codelibs.fess.crawler.entity.ResponseData;
36  import org.codelibs.fess.crawler.exception.CrawlingAccessException;
37  import org.codelibs.fess.exception.ThumbnailGenerationException;
38  import org.codelibs.fess.helper.CrawlingConfigHelper;
39  import org.codelibs.fess.helper.IndexingHelper;
40  import org.codelibs.fess.mylasta.direction.FessConfig;
41  import org.codelibs.fess.opensearch.client.SearchEngineClient;
42  import org.codelibs.fess.opensearch.config.exentity.CrawlingConfig;
43  import org.codelibs.fess.thumbnail.ThumbnailGenerator;
44  import org.codelibs.fess.util.ComponentUtil;
45  import org.codelibs.fess.util.DocumentUtil;
46  
47  /**
48   * Abstract base class for thumbnail generators.
49   * Provides common functionality for thumbnail generation implementations.
50   */
51  public abstract class BaseThumbnailGenerator implements ThumbnailGenerator {
52      private static final Logger logger = LogManager.getLogger(BaseThumbnailGenerator.class);
53  
54      /** Map of conditions for thumbnail generation. */
55      protected final Map<String, String> conditionMap = new HashMap<>();
56  
57      /** Length for directory name generation. */
58      protected int directoryNameLength = 5;
59  
60      /** List of generator names. */
61      protected List<String> generatorList;
62  
63      /** Map of file paths for thumbnail generation. */
64      protected Map<String, String> filePathMap = new HashMap<>();
65  
66      /** The name of this thumbnail generator. */
67      protected String name;
68  
69      /** Maximum number of redirects to follow. */
70      protected int maxRedirectCount = 10;
71  
72      /** Availability status of this generator. */
73      protected Boolean available = null;
74  
75      /**
76       * Registers this thumbnail generator with the thumbnail manager.
77       */
78      public void register() {
79          ComponentUtil.getThumbnailManager().add(this);
80      }
81  
82      /**
83       * Default constructor for BaseThumbnailGenerator.
84       */
85      public BaseThumbnailGenerator() {
86          // Default constructor
87      }
88  
89      /**
90       * Adds a condition for thumbnail generation.
91       * @param key The condition key.
92       * @param regex The regex pattern for the condition.
93       */
94      public void addCondition(final String key, final String regex) {
95          final String value = conditionMap.get(key);
96          if (StringUtil.isBlank(value)) {
97              conditionMap.put(key, regex);
98          } else {
99              conditionMap.put(key, value + "|" + regex);
100         }
101     }
102 
103     @Override
104     public boolean isTarget(final Map<String, Object> docMap) {
105         final String thumbnailFieldName = ComponentUtil.getFessConfig().getIndexFieldThumbnail();
106         if (logger.isDebugEnabled()) {
107             logger.debug("[{}] thumbnail: {}", name, docMap.get(thumbnailFieldName));
108         }
109         if (!docMap.containsKey(thumbnailFieldName)) {
110             return false;
111         }
112         for (final Map.Entry<String, String> entry : conditionMap.entrySet()) {
113             if (docMap.get(entry.getKey()) instanceof final String value && value.matches(entry.getValue())) {
114                 if (logger.isDebugEnabled()) {
115                     logger.debug("[{}] match {}:{}", entry.getKey(), name, value);
116                 }
117                 return true;
118             }
119         }
120         return false;
121     }
122 
123     @Override
124     public boolean isAvailable() {
125         if (available != null) {
126             return available;
127         }
128         if (generatorList != null && !generatorList.isEmpty()) {
129             String path = System.getenv("PATH");
130             if (path == null) {
131                 path = System.getenv("Path");
132             }
133             if (path == null) {
134                 path = System.getenv("path");
135             }
136             final List<String> pathList = new ArrayList<>();
137             pathList.add("/usr/share/fess/bin");
138             if (path != null) {
139                 stream(path.split(File.pathSeparator)).of(stream -> stream.map(String::trim).forEach(s -> pathList.add(s)));
140             }
141             if (logger.isDebugEnabled()) {
142                 logger.debug("search paths: {}", pathList);
143             }
144             available = generatorList.stream().map(s -> {
145                 if (s.startsWith("${path}")) {
146                     for (final String p : pathList) {
147                         final File f = new File(s.replace("${path}", p));
148                         if (f.exists()) {
149                             final String filePath = f.getAbsolutePath();
150                             filePathMap.put(s, filePath);
151                             if (logger.isDebugEnabled()) {
152                                 logger.debug("generator path: {}", filePath);
153                             }
154                             return filePath;
155                         }
156                     }
157                 }
158                 if (logger.isDebugEnabled()) {
159                     logger.debug("generator path: {}", s);
160                 }
161                 return s;
162             }).allMatch(s -> {
163                 final boolean found = new File(s).isFile();
164                 if (found && logger.isDebugEnabled()) {
165                     logger.debug("Generator command found: {}", s);
166                 }
167                 return found;
168             });
169         } else {
170             available = true;
171         }
172         return available;
173     }
174 
175     @Override
176     public Tuple3<String, String, String> createTask(final String path, final Map<String, Object> docMap) {
177         final FessConfig fessConfig = ComponentUtil.getFessConfig();
178         final String thumbnailId = DocumentUtil.getValue(docMap, fessConfig.getIndexFieldId(), String.class);
179         final Tuple3<String, String, String> task = new Tuple3<>(getName(), thumbnailId, path);
180         if (logger.isDebugEnabled()) {
181             logger.debug("Create thumbnail task: {}", task);
182         }
183         return task;
184     }
185 
186     /**
187      * Sets the directory name length for thumbnail storage.
188      * @param directoryNameLength The directory name length.
189      */
190     public void setDirectoryNameLength(final int directoryNameLength) {
191         this.directoryNameLength = directoryNameLength;
192     }
193 
194     /**
195      * Expands a file path using the file path mapping.
196      * @param value The original path value.
197      * @return The expanded path or the original value if no mapping exists.
198      */
199     protected String expandPath(final String value) {
200         if (value != null && filePathMap.containsKey(value)) {
201             return filePathMap.get(value);
202         }
203         return value;
204     }
205 
206     /**
207      * Updates the thumbnail field in the search index.
208      * @param thumbnailId The thumbnail ID.
209      * @param value The thumbnail value to update.
210      */
211     protected void updateThumbnailField(final String thumbnailId, final String value) {
212         // TODO bulk
213         final FessConfig fessConfig = ComponentUtil.getFessConfig();
214         try {
215             ComponentUtil.getIndexingHelper()
216                     .updateDocument(ComponentUtil.getSearchEngineClient(), thumbnailId, fessConfig.getIndexFieldThumbnail(), value);
217         } catch (final Exception e) {
218             logger.warn("Failed to update thumbnail field at {}", thumbnailId, e);
219         }
220     }
221 
222     /**
223      * Processes thumbnail generation with a consumer function.
224      * @param id The document ID.
225      * @param consumer The consumer function to process thumbnail and config ID.
226      * @return True if processing was successful, false otherwise.
227      */
228     protected boolean process(final String id, final BiPredicate<String, String> consumer) {
229         final FessConfig fessConfig = ComponentUtil.getFessConfig();
230         final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
231         final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
232         try {
233             final Map<String, Object> doc = indexingHelper.getDocument(searchEngineClient, id,
234                     new String[] { fessConfig.getIndexFieldThumbnail(), fessConfig.getIndexFieldConfigId() });
235             if (doc == null) {
236                 throw new ThumbnailGenerationException("Document is not found: " + id);
237             }
238             final String url = DocumentUtil.getValue(doc, fessConfig.getIndexFieldThumbnail(), String.class);
239             if (StringUtil.isBlank(url)) {
240                 throw new ThumbnailGenerationException("Invalid thumbnail: " + url);
241             }
242             final String configId = DocumentUtil.getValue(doc, fessConfig.getIndexFieldConfigId(), String.class);
243             if (configId == null || configId.length() < 2) {
244                 throw new ThumbnailGenerationException("Invalid configId: " + configId);
245             }
246             return consumer.test(configId, url);
247         } catch (final ThumbnailGenerationException e) {
248             if (e.getCause() == null) {
249                 logger.debug(e.getMessage());
250             } else {
251                 logger.warn("Failed to process thumbnail: id={}", id, e);
252             }
253         } catch (final Exception e) {
254             logger.warn("Failed to process thumbnail: id={}", id, e);
255         }
256         return false;
257     }
258 
259     /**
260      * Processes thumbnail generation with a response data consumer.
261      * @param id The document ID.
262      * @param consumer The consumer function to process response data.
263      * @return True if processing was successful, false otherwise.
264      */
265     protected boolean process(final String id, final Predicate<ResponseData> consumer) {
266         return process(id, (configId, url) -> {
267             final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
268             final CrawlingConfig config = crawlingConfigHelper.getCrawlingConfig(configId);
269             if (config == null) {
270                 throw new ThumbnailGenerationException("No CrawlingConfig: " + configId);
271             }
272 
273             if (logger.isInfoEnabled()) {
274                 logger.info("Generating Thumbnail: {}", url);
275             }
276 
277             final CrawlerClientFactory crawlerClientFactory =
278                     config.initializeClientFactory(() -> ComponentUtil.getComponent(CrawlerClientFactory.class));
279             final CrawlerClient client = crawlerClientFactory.getClient(url);
280             if (client == null) {
281                 throw new ThumbnailGenerationException("No CrawlerClient: " + configId + ", url: " + url);
282             }
283             String u = url;
284             for (int i = 0; i < maxRedirectCount; i++) {
285                 try (final ResponseData responseData = client.execute(RequestDataBuilder.newRequestData().get().url(u).build())) {
286                     if (StringUtil.isNotBlank(responseData.getRedirectLocation())) {
287                         u = responseData.getRedirectLocation();
288                         continue;
289                     }
290                     if (StringUtil.isBlank(responseData.getUrl())) {
291                         throw new ThumbnailGenerationException(
292                                 "Failed to process a thumbnail content: " + url + " (Response URL is empty)");
293                     }
294                     return consumer.test(responseData);
295                 } catch (final CrawlingAccessException e) {
296                     if (logger.isDebugEnabled()) {
297                         throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
298                     }
299                     throw new ThumbnailGenerationException(e.getMessage());
300                 } catch (final Exception e) {
301                     throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url, e);
302                 }
303             }
304             throw new ThumbnailGenerationException("Failed to process a thumbnail content: " + url + " (Redirect Loop)");
305         });
306     }
307 
308     /**
309      * Sets the list of generator names.
310      * @param generatorList The list of generator names.
311      */
312     public void setGeneratorList(final List<String> generatorList) {
313         this.generatorList = generatorList;
314     }
315 
316     @Override
317     public String getName() {
318         return name;
319     }
320 
321     /**
322      * Sets the name of this thumbnail generator.
323      * @param name The generator name.
324      */
325     public void setName(final String name) {
326         this.name = name;
327     }
328 
329     /**
330      * Sets the maximum number of redirects to follow.
331      * @param maxRedirectCount The maximum redirect count.
332      */
333     public void setMaxRedirectCount(final int maxRedirectCount) {
334         this.maxRedirectCount = maxRedirectCount;
335     }
336 
337 }